Merge branch 'feature/cale/cms-master' into feature/christina/misc

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "common/test/phantom-jasmine"]
-	path = common/test/phantom-jasmine
-	url = https://github.com/jcarver989/phantom-jasmine.git

apt-packages.txt

@@ -0,0 +1,25 @@
+python-software-properties
+pkg-config
+curl
+git
+python-virtualenv
+build-essential
+python-dev
+gfortran
+liblapack-dev
+libfreetype6-dev
+libpng12-dev
+libxml2-dev
+libxslt-dev
+yui-compressor
+graphviz
+graphviz-dev
+mysql-server
+libmysqlclient-dev
+libgeos-dev
+libreadline6
+libreadline6-dev
+mongodb
+nodejs
+npm
+coffeescript

apt-repos.txt

@@ -0,0 +1,3 @@
+ppa:chris-lea/node.js
+ppa:chris-lea/node.js-libs
+ppa:chris-lea/libjs-underscore

brew-formulas.txt

@@ -1,10 +1,12 @@
-readline 
-sqlite 
-gdbm 
-pkg-config 
-gfortran 
-python 
-yuicompressor 
+readline
+sqlite
+gdbm
+pkg-config
+gfortran
+python
+yuicompressor
 node
 graphviz
 mysql
+geos
+mongodb

cms/.coveragerc

@@ -2,11 +2,13 @@
 [run]
 data_file = reports/cms/.coverage
 source = cms
+omit = cms/envs/*, cms/manage.py
 
 [report]
 ignore_errors = True
 
 [html]
+title = CMS Python Test Coverage Report
 directory = reports/cms/cover
 
 [xml]

common/djangoapps/mitxmako/shortcuts.py

@@ -12,10 +12,6 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-import logging
-
-log = logging.getLogger("mitx." + __name__)
-
 from django.template import Context
 from django.http import HttpResponse
 

common/djangoapps/mitxmako/template.py

@@ -54,5 +54,4 @@ class Template(MakoTemplate):
         context_dictionary['MITX_ROOT_URL'] = settings.MITX_ROOT_URL
         context_dictionary['django_context'] = context_instance
                 
-        return super(Template, self).render(**context_dictionary)
-        
+        return super(Template, self).render_unicode(**context_dictionary)

common/djangoapps/student/models.py

@@ -36,10 +36,12 @@ file and check it in at the same time as your model changes. To do that,
 3. Add the migration file created in mitx/common/djangoapps/student/migrations/
 """
 from datetime import datetime
+import hashlib
 import json
 import logging
 import uuid
 
+
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.db import models
@@ -47,7 +49,6 @@ from django.db.models.signals import post_save
 from django.dispatch import receiver
 
 import comment_client as cc
-from django_comment_client.models import Role
 
 
 log = logging.getLogger(__name__)
@@ -125,9 +126,9 @@ class UserProfile(models.Model):
         self.meta = json.dumps(js)
 
 class TestCenterUser(models.Model):
-    """This is our representation of the User for in-person testing, and 
+    """This is our representation of the User for in-person testing, and
     specifically for Pearson at this point. A few things to note:
-    
+
     * Pearson only supports Latin-1, so we have to make sure that the data we
       capture here will work with that encoding.
     * While we have a lot of this demographic data in UserProfile, it's much
@@ -135,9 +136,9 @@ class TestCenterUser(models.Model):
       UserProfile, but we'll need to have a step where people who are signing
       up re-enter their demographic data into the fields we specify.
     * Users are only created here if they register to take an exam in person.
-    
+
     The field names and lengths are modeled on the conventions and constraints
-    of Pearson's data import system, including oddities such as suffix having 
+    of Pearson's data import system, including oddities such as suffix having
     a limit of 255 while last_name only gets 50.
     """
     # Our own record keeping...
@@ -148,21 +149,21 @@ class TestCenterUser(models.Model):
     # and is something Pearson needs to know to manage updates. Unlike
     # updated_at, this will not get incremented when we do a batch data import.
     user_updated_at = models.DateTimeField(db_index=True)
-    
+
     # Unique ID given to us for this User by the Testing Center. It's null when
     # we first create the User entry, and is assigned by Pearson later.
     candidate_id = models.IntegerField(null=True, db_index=True)
-    
+
     # Unique ID we assign our user for a the Test Center.
     client_candidate_id = models.CharField(max_length=50, db_index=True)
-    
+
     # Name
     first_name = models.CharField(max_length=30, db_index=True)
     last_name = models.CharField(max_length=50, db_index=True)
     middle_name = models.CharField(max_length=30, blank=True)
     suffix = models.CharField(max_length=255, blank=True)
     salutation = models.CharField(max_length=50, blank=True)
-    
+
     # Address
     address_1 = models.CharField(max_length=40)
     address_2 = models.CharField(max_length=40, blank=True)
@@ -175,7 +176,7 @@ class TestCenterUser(models.Model):
     postal_code = models.CharField(max_length=16, blank=True, db_index=True)
     # country is a ISO 3166-1 alpha-3 country code (e.g. "USA", "CAN", "MNG")
     country = models.CharField(max_length=3, db_index=True)
-    
+
     # Phone
     phone = models.CharField(max_length=35)
     extension = models.CharField(max_length=8, blank=True, db_index=True)
@@ -183,14 +184,27 @@ class TestCenterUser(models.Model):
     fax = models.CharField(max_length=35, blank=True)
     # fax_country_code required *if* fax is present.
     fax_country_code = models.CharField(max_length=3, blank=True)
-    
+
     # Company
     company_name = models.CharField(max_length=50, blank=True)
-    
+
     @property
     def email(self):
         return self.user.email
 
+def unique_id_for_user(user):
+    """
+    Return a unique id for a user, suitable for inserting into
+    e.g. personalized survey links.
+    """
+    # include the secret key as a salt, and to make the ids unique accross
+    # different LMS installs.    
+    h = hashlib.md5()
+    h.update(settings.SECRET_KEY)
+    h.update(str(user.id))
+    return h.hexdigest()
+
+
 ## TODO: Should be renamed to generic UserGroup, and possibly
 # Given an optional field for type of group
 class UserTestGroup(models.Model):
@@ -247,15 +261,6 @@ class CourseEnrollment(models.Model):
         return "[CourseEnrollment] %s: %s (%s)" % (self.user, self.course_id, self.created)
 
 
-@receiver(post_save, sender=CourseEnrollment)
-def assign_default_role(sender, instance, **kwargs):
-    if instance.user.is_staff:
-        role = Role.objects.get_or_create(course_id=instance.course_id, name="Moderator")[0]
-    else:
-        role = Role.objects.get_or_create(course_id=instance.course_id, name="Student")[0]
-
-    logging.info("assign_default_role: adding %s as %s" % (instance.user, role))
-    instance.user.roles.add(role)
 
 #cache_relation(User.profile)
 
@@ -363,10 +368,10 @@ def replicate_user_save(sender, **kwargs):
 
 # @receiver(post_save, sender=CourseEnrollment)
 def replicate_enrollment_save(sender, **kwargs):
-    """This is called when a Student enrolls in a course. It has to do the 
+    """This is called when a Student enrolls in a course. It has to do the
     following:
 
-    1. Make sure the User is copied into the Course DB. It may already exist 
+    1. Make sure the User is copied into the Course DB. It may already exist
        (someone deleting and re-adding a course). This has to happen first or
        the foreign key constraint breaks.
     2. Replicate the CourseEnrollment.
@@ -410,9 +415,9 @@ USER_FIELDS_TO_COPY = ["id", "username", "first_name", "last_name", "email",
 
 def replicate_user(portal_user, course_db_name):
     """Replicate a User to the correct Course DB. This is more complicated than
-    it should be because Askbot extends the auth_user table and adds its own 
+    it should be because Askbot extends the auth_user table and adds its own
     fields. So we need to only push changes to the standard fields and leave
-    the rest alone so that Askbot changes at the Course DB level don't get 
+    the rest alone so that Askbot changes at the Course DB level don't get
     overridden.
     """
     try:
@@ -457,7 +462,7 @@ def is_valid_course_id(course_id):
     """Right now, the only database that's not a course database is 'default'.
     I had nicer checking in here originally -- it would scan the courses that
     were in the system and only let you choose that. But it was annoying to run
-    tests with, since we don't have course data for some for our course test 
+    tests with, since we don't have course data for some for our course test
     databases. Hence the lazy version.
     """
     return course_id != 'default'

common/djangoapps/student/tests.py

@@ -6,11 +6,16 @@ Replace this with more appropriate tests for your application.
 """
 import logging
 from datetime import datetime
+from hashlib import sha1
 
 from django.test import TestCase
+from mock import patch, Mock
 from nose.plugins.skip import SkipTest
 
-from .models import User, UserProfile, CourseEnrollment, replicate_user, USER_FIELDS_TO_COPY
+from .models import (User, UserProfile, CourseEnrollment,
+                     replicate_user, USER_FIELDS_TO_COPY,
+                     unique_id_for_user)
+from .views import process_survey_link, _cert_info
 
 COURSE_1 = 'edX/toy/2012_Fall'
 COURSE_2 = 'edx/full/6.002_Spring_2012'
@@ -55,7 +60,7 @@ class ReplicationTest(TestCase):
         # This hasattr lameness is here because we don't want this test to be
         # triggered when we're being run by CMS tests (Askbot doesn't exist
         # there, so the test will fail).
-        # 
+        #
         # seen_response_count isn't a field we care about, so it shouldn't have
         # been copied over.
         if hasattr(portal_user, 'seen_response_count'):
@@ -74,7 +79,7 @@ class ReplicationTest(TestCase):
 
         # During this entire time, the user data should never have made it over
         # to COURSE_2
-        self.assertRaises(User.DoesNotExist, 
+        self.assertRaises(User.DoesNotExist,
                           User.objects.using(COURSE_2).get,
                           id=portal_user.id)
 
@@ -108,19 +113,19 @@ class ReplicationTest(TestCase):
         # Grab all the copies we expect
         course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
         self.assertEquals(portal_user, course_user)
-        self.assertRaises(User.DoesNotExist, 
+        self.assertRaises(User.DoesNotExist,
                           User.objects.using(COURSE_2).get,
                           id=portal_user.id)
 
         course_enrollment = CourseEnrollment.objects.using(COURSE_1).get(id=portal_enrollment.id)
         self.assertEquals(portal_enrollment, course_enrollment)
-        self.assertRaises(CourseEnrollment.DoesNotExist, 
+        self.assertRaises(CourseEnrollment.DoesNotExist,
                           CourseEnrollment.objects.using(COURSE_2).get,
                           id=portal_enrollment.id)
 
         course_user_profile = UserProfile.objects.using(COURSE_1).get(id=portal_user_profile.id)
         self.assertEquals(portal_user_profile, course_user_profile)
-        self.assertRaises(UserProfile.DoesNotExist, 
+        self.assertRaises(UserProfile.DoesNotExist,
                           UserProfile.objects.using(COURSE_2).get,
                           id=portal_user_profile.id)
 
@@ -174,30 +179,112 @@ class ReplicationTest(TestCase):
         portal_user.save()
         portal_user_profile.gender = 'm'
         portal_user_profile.save()
-      
-        # Grab all the copies we expect, and make sure it doesn't end up in 
+
+        # Grab all the copies we expect, and make sure it doesn't end up in
         # places we don't expect.
         course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
         self.assertEquals(portal_user, course_user)
-        self.assertRaises(User.DoesNotExist, 
+        self.assertRaises(User.DoesNotExist,
                           User.objects.using(COURSE_2).get,
                           id=portal_user.id)
 
         course_enrollment = CourseEnrollment.objects.using(COURSE_1).get(id=portal_enrollment.id)
         self.assertEquals(portal_enrollment, course_enrollment)
-        self.assertRaises(CourseEnrollment.DoesNotExist, 
+        self.assertRaises(CourseEnrollment.DoesNotExist,
                           CourseEnrollment.objects.using(COURSE_2).get,
                           id=portal_enrollment.id)
 
         course_user_profile = UserProfile.objects.using(COURSE_1).get(id=portal_user_profile.id)
         self.assertEquals(portal_user_profile, course_user_profile)
-        self.assertRaises(UserProfile.DoesNotExist, 
+        self.assertRaises(UserProfile.DoesNotExist,
                           UserProfile.objects.using(COURSE_2).get,
                           id=portal_user_profile.id)
 
 
+class CourseEndingTest(TestCase):
+    """Test things related to course endings: certificates, surveys, etc"""
 
+    def test_process_survey_link(self):
+        username = "fred"
+        user = Mock(username=username)
+        id = unique_id_for_user(user)
+        link1 = "http://www.mysurvey.com"
+        self.assertEqual(process_survey_link(link1, user), link1)
 
+        link2 = "http://www.mysurvey.com?unique={UNIQUE_ID}"
+        link2_expected = "http://www.mysurvey.com?unique={UNIQUE_ID}".format(UNIQUE_ID=id)
+        self.assertEqual(process_survey_link(link2, user), link2_expected)
 
+    def test_cert_info(self):
+        user = Mock(username="fred")
+        survey_url = "http://a_survey.com"
+        course = Mock(end_of_course_survey_url=survey_url)
 
+        self.assertEqual(_cert_info(user, course, None),
+                         {'status': 'processing',
+                          'show_disabled_download_button': False,
+                          'show_download_url': False,
+                          'show_survey_button': False,})
 
+        cert_status = {'status': 'unavailable'}
+        self.assertEqual(_cert_info(user, course, cert_status),
+                         {'status': 'processing',
+                          'show_disabled_download_button': False,
+                          'show_download_url': False,
+                          'show_survey_button': False})
+
+        cert_status = {'status': 'generating', 'grade': '67'}
+        self.assertEqual(_cert_info(user, course, cert_status),
+                         {'status': 'generating',
+                          'show_disabled_download_button': True,
+                          'show_download_url': False,
+                          'show_survey_button': True,
+                          'survey_url': survey_url,
+                          'grade': '67'
+                          })
+
+        cert_status = {'status': 'regenerating', 'grade': '67'}
+        self.assertEqual(_cert_info(user, course, cert_status),
+                         {'status': 'generating',
+                          'show_disabled_download_button': True,
+                          'show_download_url': False,
+                          'show_survey_button': True,
+                          'survey_url': survey_url,
+                          'grade': '67'
+                          })
+
+        download_url = 'http://s3.edx/cert'
+        cert_status = {'status': 'downloadable', 'grade': '67',
+                       'download_url': download_url}
+        self.assertEqual(_cert_info(user, course, cert_status),
+                         {'status': 'ready',
+                          'show_disabled_download_button': False,
+                          'show_download_url': True,
+                          'download_url': download_url,
+                          'show_survey_button': True,
+                          'survey_url': survey_url,
+                          'grade': '67'
+                          })
+
+        cert_status = {'status': 'notpassing', 'grade': '67',
+                       'download_url': download_url}
+        self.assertEqual(_cert_info(user, course, cert_status),
+                         {'status': 'notpassing',
+                          'show_disabled_download_button': False,
+                          'show_download_url': False,
+                          'show_survey_button': True,
+                          'survey_url': survey_url,
+                          'grade': '67'
+                          })
+
+        # Test a course that doesn't have a survey specified
+        course2 = Mock(end_of_course_survey_url=None)
+        cert_status = {'status': 'notpassing', 'grade': '67',
+                       'download_url': download_url}
+        self.assertEqual(_cert_info(user, course2, cert_status),
+                         {'status': 'notpassing',
+                          'show_disabled_download_button': False,
+                          'show_download_url': False,
+                          'show_survey_button': False,
+                          'grade': '67'
+                          })

common/djangoapps/student/views.py

@@ -28,7 +28,7 @@ from django.core.cache import cache
 from django_future.csrf import ensure_csrf_cookie, csrf_exempt
 from student.models import (Registration, UserProfile,
                             PendingNameChange, PendingEmailChange,
-                            CourseEnrollment)
+                            CourseEnrollment, unique_id_for_user)
 
 from certificates.models import CertificateStatuses, certificate_status_for_student
 
@@ -39,7 +39,8 @@ from xmodule.modulestore.exceptions import ItemNotFoundError
 
 from datetime import date
 from collections import namedtuple
-from courseware.courses import get_courses_by_university
+
+from courseware.courses import get_courses
 from courseware.access import has_access
 
 from statsd import statsd
@@ -68,31 +69,26 @@ def index(request, extra_context={}, user=None):
     extra_context is used to allow immediate display of certain modal windows, eg signup,
     as used by external_auth.
     '''
-    feed_data = cache.get("students_index_rss_feed_data")
-    if feed_data == None:
-        if hasattr(settings, 'RSS_URL'):
-            feed_data = urllib.urlopen(settings.RSS_URL).read()
-        else:
-            feed_data = render_to_string("feed.rss", None)
-        cache.set("students_index_rss_feed_data", feed_data, settings.RSS_TIMEOUT)
-
-    feed = feedparser.parse(feed_data)
-    entries = feed['entries'][0:3]
-    for entry in entries:
-        soup = BeautifulSoup(entry.description)
-        entry.image = soup.img['src'] if soup.img else None
-        entry.summary = soup.getText()
 
     # The course selection work is done in courseware.courses.
     domain = settings.MITX_FEATURES.get('FORCE_UNIVERSITY_DOMAIN')	# normally False
     if domain==False:				# do explicit check, because domain=None is valid
         domain = request.META.get('HTTP_HOST')
-    universities = get_courses_by_university(None,
-                                             domain=domain)
-    context = {'universities': universities, 'entries': entries}
+
+    courses = get_courses(None, domain=domain)
+
+    # Sort courses by how far are they from they start day
+    key = lambda course: course.metadata['days_to_start']
+    courses = sorted(courses, key=key, reverse=True)
+
+    # Get the 3 most recent news
+    top_news = _get_news(top=3)
+
+    context = {'courses': courses, 'news': top_news}
     context.update(extra_context)
     return render_to_response('index.html', context)
 
+
 def course_from_id(course_id):
     """Return the CourseDescriptor corresponding to this course_id"""
     course_loc = CourseDescriptor.id_to_location(course_id)
@@ -107,9 +103,9 @@ def get_date_for_press(publish_date):
     # strip off extra months, and just use the first:
     date = re.sub(multimonth_pattern, ", ", publish_date)
     if re.search(day_pattern, date):
-        date = datetime.datetime.strptime(date, "%B %d, %Y") 
-    else: 
-        date = datetime.datetime.strptime(date, "%B, %Y") 
+        date = datetime.datetime.strptime(date, "%B %d, %Y")
+    else:
+        date = datetime.datetime.strptime(date, "%B, %Y")
     return date
 
 def press(request):
@@ -127,6 +123,87 @@ def press(request):
     return render_to_response('static_templates/press.html', {'articles': articles})
 
 
+def process_survey_link(survey_link, user):
+    """
+    If {UNIQUE_ID} appears in the link, replace it with a unique id for the user.
+    Currently, this is sha1(user.username).  Otherwise, return survey_link.
+    """
+    return survey_link.format(UNIQUE_ID=unique_id_for_user(user))
+
+
+def cert_info(user, course):
+    """
+    Get the certificate info needed to render the dashboard section for the given
+    student and course.  Returns a dictionary with keys:
+
+    'status': one of 'generating', 'ready', 'notpassing', 'processing'
+    'show_download_url': bool
+    'download_url': url, only present if show_download_url is True
+    'show_disabled_download_button': bool -- true if state is 'generating'
+    'show_survey_button': bool
+    'survey_url': url, only if show_survey_button is True
+    'grade': if status is not 'processing'
+    """
+    if not course.has_ended():
+        return {}
+
+    return _cert_info(user, course, certificate_status_for_student(user, course.id))
+
+def _cert_info(user, course, cert_status):
+    """
+    Implements the logic for cert_info -- split out for testing.
+    """
+    default_status = 'processing'
+
+    default_info = {'status': default_status,
+                    'show_disabled_download_button': False,
+                    'show_download_url': False,
+                    'show_survey_button': False}
+
+    if cert_status is None:
+        return default_info
+
+    # simplify the status for the template using this lookup table
+    template_state = {
+        CertificateStatuses.generating: 'generating',
+        CertificateStatuses.regenerating: 'generating',
+        CertificateStatuses.downloadable: 'ready',
+        CertificateStatuses.notpassing: 'notpassing',
+        }
+
+    status = template_state.get(cert_status['status'], default_status)
+
+    d = {'status': status,
+         'show_download_url': status == 'ready',
+         'show_disabled_download_button': status == 'generating',}
+
+    if (status in ('generating', 'ready', 'notpassing') and
+        course.end_of_course_survey_url is not None):
+        d.update({
+         'show_survey_button': True,
+         'survey_url': process_survey_link(course.end_of_course_survey_url, user)})
+    else:
+        d['show_survey_button'] = False
+
+    if status == 'ready':
+        if 'download_url' not in cert_status:
+            log.warning("User %s has a downloadable cert for %s, but no download url",
+                        user.username, course.id)
+            return default_info
+        else:
+            d['download_url'] = cert_status['download_url']
+
+    if status in ('generating', 'ready', 'notpassing'):
+        if 'grade' not in cert_status:
+            # Note: as of 11/20/2012, we know there are students in this state-- cs169.1x,
+            # who need to be regraded (we weren't tracking 'notpassing' at first).
+            # We can add a log.warning here once we think it shouldn't happen.
+            return default_info
+        else:
+            d['grade'] = cert_status['grade']
+
+    return d
+
 @login_required
 @ensure_csrf_cookie
 def dashboard(request):
@@ -160,12 +237,10 @@ def dashboard(request):
     show_courseware_links_for = frozenset(course.id for course in courses
                                           if has_access(request.user, course, 'load'))
 
-    # TODO: workaround to not have to zip courses and certificates in the template
-    # since before there is a migration to certificates
-    if settings.MITX_FEATURES.get('CERTIFICATES_ENABLED'):
-        cert_statuses = { course.id: certificate_status_for_student(request.user, course.id) for course in courses}
-    else:
-        cert_statuses = {}
+    cert_statuses = { course.id: cert_info(request.user, course) for course in courses}
+
+    # Get the 3 most recent news
+    top_news = _get_news(top=3)
 
     context = {'courses': courses,
                'message': message,
@@ -173,6 +248,7 @@ def dashboard(request):
                'errored_courses': errored_courses,
                'show_courseware_links_for' : show_courseware_links_for,
                'cert_statuses': cert_statuses,
+               'news': top_news,
                }
 
     return render_to_response('dashboard.html', context)
@@ -262,6 +338,14 @@ def change_enrollment(request):
     return {'success': False, 'error': 'We weren\'t able to unenroll you. Please try again.'}
 
 
+@ensure_csrf_cookie
+def accounts_login(request, error=""):
+
+
+    return render_to_response('accounts_login.html', { 'error': error })
+
+
+
 # Need different levels of logging
 @ensure_csrf_cookie
 def login_user(request, error=""):
@@ -820,3 +904,24 @@ def test_center_login(request):
         return redirect('/courses/MITx/6.002x/2012_Fall/courseware/Final_Exam/Final_Exam_Fall_2012/')
     else:
         return HttpResponseForbidden()
+
+
+def _get_news(top=None):
+    "Return the n top news items on settings.RSS_URL"
+
+    feed_data = cache.get("students_index_rss_feed_data")
+    if feed_data == None:
+        if hasattr(settings, 'RSS_URL'):
+            feed_data = urllib.urlopen(settings.RSS_URL).read()
+        else:
+            feed_data = render_to_string("feed.rss", None)
+        cache.set("students_index_rss_feed_data", feed_data, settings.RSS_TIMEOUT)
+
+    feed = feedparser.parse(feed_data)
+    entries = feed['entries'][0:top]  # all entries if top is None
+    for entry in entries:
+        soup = BeautifulSoup(entry.description)
+        entry.image = soup.img['src'] if soup.img else None
+        entry.summary = soup.getText()
+
+    return entries

common/djangoapps/track/migrations/0001_initial.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'TrackingLog'
+        db.create_table('track_trackinglog', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('dtcreated', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+            ('username', self.gf('django.db.models.fields.CharField')(max_length=32, blank=True)),
+            ('ip', self.gf('django.db.models.fields.CharField')(max_length=32, blank=True)),
+            ('event_source', self.gf('django.db.models.fields.CharField')(max_length=32)),
+            ('event_type', self.gf('django.db.models.fields.CharField')(max_length=32, blank=True)),
+            ('event', self.gf('django.db.models.fields.TextField')(blank=True)),
+            ('agent', self.gf('django.db.models.fields.CharField')(max_length=256, blank=True)),
+            ('page', self.gf('django.db.models.fields.CharField')(max_length=32, null=True, blank=True)),
+            ('time', self.gf('django.db.models.fields.DateTimeField')()),
+        ))
+        db.send_create_signal('track', ['TrackingLog'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'TrackingLog'
+        db.delete_table('track_trackinglog')
+
+
+    models = {
+        'track.trackinglog': {
+            'Meta': {'object_name': 'TrackingLog'},
+            'agent': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
+            'dtcreated': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'event': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'event_source': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
+            'event_type': ('django.db.models.fields.CharField', [], {'max_length': '32', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'ip': ('django.db.models.fields.CharField', [], {'max_length': '32', 'blank': 'True'}),
+            'page': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True', 'blank': 'True'}),
+            'time': ('django.db.models.fields.DateTimeField', [], {}),
+            'username': ('django.db.models.fields.CharField', [], {'max_length': '32', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['track']

common/djangoapps/track/migrations/0002_auto__add_field_trackinglog_host__chg_field_trackinglog_event_type__ch.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'TrackingLog.host'
+        db.add_column('track_trackinglog', 'host',
+                      self.gf('django.db.models.fields.CharField')(default='', max_length=64, blank=True),
+                      keep_default=False)
+
+
+        # Changing field 'TrackingLog.event_type'
+        db.alter_column('track_trackinglog', 'event_type', self.gf('django.db.models.fields.CharField')(max_length=512))
+
+        # Changing field 'TrackingLog.page'
+        db.alter_column('track_trackinglog', 'page', self.gf('django.db.models.fields.CharField')(max_length=512, null=True))
+
+    def backwards(self, orm):
+        # Deleting field 'TrackingLog.host'
+        db.delete_column('track_trackinglog', 'host')
+
+
+        # Changing field 'TrackingLog.event_type'
+        db.alter_column('track_trackinglog', 'event_type', self.gf('django.db.models.fields.CharField')(max_length=32))
+
+        # Changing field 'TrackingLog.page'
+        db.alter_column('track_trackinglog', 'page', self.gf('django.db.models.fields.CharField')(max_length=32, null=True))
+
+    models = {
+        'track.trackinglog': {
+            'Meta': {'object_name': 'TrackingLog'},
+            'agent': ('django.db.models.fields.CharField', [], {'max_length': '256', 'blank': 'True'}),
+            'dtcreated': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'event': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'event_source': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
+            'event_type': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'host': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'ip': ('django.db.models.fields.CharField', [], {'max_length': '32', 'blank': 'True'}),
+            'page': ('django.db.models.fields.CharField', [], {'max_length': '512', 'null': 'True', 'blank': 'True'}),
+            'time': ('django.db.models.fields.DateTimeField', [], {}),
+            'username': ('django.db.models.fields.CharField', [], {'max_length': '32', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['track']

common/djangoapps/track/migrations/__init__.py

@@ -0,0 +1 @@
+

common/djangoapps/track/models.py

@@ -7,11 +7,12 @@ class TrackingLog(models.Model):
     username = models.CharField(max_length=32,blank=True)
     ip = models.CharField(max_length=32,blank=True)
     event_source = models.CharField(max_length=32)
-    event_type = models.CharField(max_length=32,blank=True)
+    event_type = models.CharField(max_length=512,blank=True)
     event = models.TextField(blank=True)
     agent = models.CharField(max_length=256,blank=True)
-    page = models.CharField(max_length=32,blank=True,null=True)
+    page = models.CharField(max_length=512,blank=True,null=True)
     time = models.DateTimeField('event time')
+    host = models.CharField(max_length=64,blank=True)
 
     def __unicode__(self):
         s = "[%s] %s@%s: %s | %s | %s | %s" % (self.time, self.username, self.ip, self.event_source,

common/djangoapps/track/views.py

@@ -17,7 +17,7 @@ from track.models import TrackingLog
 
 log = logging.getLogger("tracking")
 
-LOGFIELDS = ['username','ip','event_source','event_type','event','agent','page','time']
+LOGFIELDS = ['username','ip','event_source','event_type','event','agent','page','time','host']
 
 def log_event(event):
     event_str = json.dumps(event)
@@ -58,6 +58,7 @@ def user_track(request):
         "agent": agent,
         "page": request.GET['page'],
         "time": datetime.datetime.utcnow().isoformat(),
+        "host": request.META['SERVER_NAME'],
         }
     log_event(event)
     return HttpResponse('success')
@@ -83,6 +84,7 @@ def server_track(request, event_type, event, page=None):
         "agent": agent,
         "page": page,
         "time": datetime.datetime.utcnow().isoformat(),
+        "host": request.META['SERVER_NAME'],
         }
 
     if event_type.startswith("/event_logs") and request.user.is_staff:	# don't log

common/djangoapps/util/json_request.py

@@ -4,6 +4,11 @@ import json
 
 
 def expect_json(view_function):
+    """
+    View decorator for simplifying handing of requests that expect json.  If the request's
+    CONTENT_TYPE is application/json, parses the json dict from request.body, and updates
+    request.POST with the contents.
+    """
     @wraps(view_function)
     def expect_json_with_cloned_request(request, *args, **kwargs):
         # cdodge: fix postback errors in CMS. The POST 'content-type' header can include additional information

common/lib/capa/.coveragerc

@@ -7,6 +7,7 @@ source = common/lib/capa
 ignore_errors = True
 
 [html]
+title = Capa Python Test Coverage Report
 directory = reports/common/lib/capa/cover
 
 [xml]

common/lib/capa/capa/capa_problem.py

@@ -33,6 +33,7 @@ from xml.sax.saxutils import unescape
 import chem
 import chem.chemcalc
 import chem.chemtools
+import chem.miller
 
 import calc
 from correctmap import CorrectMap
@@ -52,7 +53,7 @@ response_tag_dict = dict([(x.response_tag, x) for x in responsetypes.__all__])
 solution_tags = ['solution']
 
 # these get captured as student responses
-response_properties = ["codeparam", "responseparam", "answer"]
+response_properties = ["codeparam", "responseparam", "answer", "openendedparam"]
 
 # special problem tags which should be turned into innocuous HTML
 html_transforms = {'problem': {'tag': 'div'},
@@ -67,10 +68,11 @@ global_context = {'random': random,
                   'calc': calc,
                   'eia': eia,
                   'chemcalc': chem.chemcalc,
-                  'chemtools': chem.chemtools}
+                  'chemtools': chem.chemtools,
+                  'miller': chem.miller}
 
 # These should be removed from HTML output, including all subelements
-html_problem_semantics = ["codeparam", "responseparam", "answer", "script", "hintgroup"]
+html_problem_semantics = ["codeparam", "responseparam", "answer", "script", "hintgroup", "openendedparam","openendedrubric"]
 
 log = logging.getLogger('mitx.' + __name__)
 

common/lib/capa/capa/chem/miller.py

@@ -0,0 +1,267 @@
+""" Calculation of Miller indices """
+
+import numpy as np
+import math
+import fractions as fr
+import decimal
+import json
+
+
+def lcm(a, b):
+    """
+    Returns least common multiple of a, b
+
+    Args:
+        a, b: floats
+
+    Returns:
+        float
+    """
+    return a * b / fr.gcd(a, b)
+
+
+def segment_to_fraction(distance):
+    """
+    Converts lengths of which the plane cuts the axes to fraction.
+
+    Tries convert distance to closest nicest fraction with denominator less or
+    equal than 10. It is
+    purely for simplicity and clearance of learning purposes. Jenny: 'In typical
+    courses students usually do not encounter indices any higher than 6'.
+
+    If distance is not a number (numpy nan), it means that plane is parallel to
+    axis or contains it. Inverted fraction to nan (nan is 1/0) = 0 / 1 is
+    returned
+
+    Generally (special cases):
+
+    a) if distance is smaller than some constant, i.g. 0.01011,
+    than fraction's denominator usually much greater than 10.
+
+    b) Also, if student will set point on 0.66 -> 1/3, so it is 333 plane,
+    But if he will slightly move the mouse and click on 0.65 -> it will be
+    (16,15,16) plane. That's why we are doing adjustments for points coordinates,
+    to the closest tick, tick + tick / 2 value. And now UI sends to server only
+    values multiple to 0.05  (half of tick). Same rounding is implemented for
+    unittests.
+
+    But if one will want to calculate miller indices with exact coordinates and
+    with nice fractions (which produce small Miller indices), he may want shift
+    to new origin if segments are like S = (0.015, > 0.05, >0.05) - close to zero
+    in one coordinate. He may update S to (0, >0.05, >0.05) and shift origin.
+    In this way he can recieve nice small fractions. Also there is can be
+    degenerated case when S = (0.015, 0.012, >0.05) - if update S to (0, 0, >0.05) -
+    it is a line. This case  should be considered separately. Small nice Miller
+    numbers and possibility to create very small segments can not be implemented
+    at same time).
+
+
+    Args:
+        distance: float distance that plane cuts on axis, it must not be 0.
+        Distance is multiple of 0.05.
+
+    Returns:
+         Inverted fraction.
+         0 / 1 if distance is nan
+
+    """
+    if np.isnan(distance):
+        return fr.Fraction(0, 1)
+    else:
+        fract = fr.Fraction(distance).limit_denominator(10)
+        return fr.Fraction(fract.denominator, fract.numerator)
+
+
+def sub_miller(segments):
+    '''
+    Calculates Miller indices from segments.
+
+    Algorithm:
+
+    1. Obtain inverted fraction from segments
+
+    2. Find common denominator of inverted fractions
+
+    3. Lead fractions to common denominator and throws denominator away.
+
+    4. Return obtained values.
+
+    Args:
+        List of 3 floats, meaning distances that plane cuts on x, y, z axes.
+        Any float not equals zero, it means that plane does not intersect origin,
+        i. e. shift of origin has already been done.
+
+    Returns:
+        String that represents Miller indices, e.g: (-6,3,-6) or (2,2,2)
+    '''
+    fracts = [segment_to_fraction(segment) for segment in segments]
+    common_denominator = reduce(lcm, [fract.denominator for fract in fracts])
+    miller = ([fract.numerator * math.fabs(common_denominator) /
+                                fract.denominator for fract in fracts])
+    return'(' + ','.join(map(str, map(decimal.Decimal, miller))) + ')'
+
+
+def miller(points):
+    """
+    Calculates Miller indices from points.
+
+    Algorithm:
+
+    1. Calculate normal vector to a plane that goes trough all points.
+
+    2. Set origin.
+
+    3. Create Cartesian coordinate system (Ccs).
+
+    4. Find the lengths of segments of which the plane cuts the axes. Equation
+       of a line for axes: Origin + (Coordinate_vector - Origin) * parameter.
+
+    5. If plane goes trough Origin:
+
+       a) Find new random origin: find unit cube vertex, not crossed by a plane.
+
+       b) Repeat 2-4.
+
+       c) Fix signs of segments after Origin shift. This means to consider
+          original directions of axes. I.g.: Origin was 0,0,0 and became
+          new_origin. If new_origin has same Y coordinate as Origin, then segment
+          does not change its sign. But if new_origin has another Y coordinate than
+          origin (was 0, became 1), than segment has to change its sign (it now
+          lies on negative side of Y axis). New Origin 0 value of X or Y or Z
+          coordinate means that segment does not change sign, 1 value -> does
+          change. So new sign  is (1 - 2 * new_origin): 0 -> 1, 1 -> -1
+
+    6. Run function that calculates miller indices from segments.
+
+    Args:
+        List of points. Each point is list of float coordinates. Order of
+        coordinates in point's list: x, y, z. Points are different!
+
+    Returns:
+        String that represents Miller indices, e.g: (-6,3,-6) or (2,2,2)
+    """
+
+    N = np.cross(points[1] - points[0], points[2] - points[0])
+    O = np.array([0, 0, 0])
+    P = points[0]  # point of plane
+    Ccs = map(np.array, [[1.0, 0, 0], [0, 1.0, 0], [0, 0, 1.0]])
+    segments = ([np.dot(P - O, N) / np.dot(ort, N) if np.dot(ort, N) != 0 else
+                                            np.nan for ort in Ccs])
+    if any(x == 0 for x in segments):  # Plane goes through origin.
+        vertices = [  # top:
+                    np.array([1.0, 1.0, 1.0]),
+                    np.array([0.0, 0.0, 1.0]),
+                    np.array([1.0, 0.0, 1.0]),
+                    np.array([0.0, 1.0, 1.0]),
+                    # bottom, except 0,0,0:
+                    np.array([1.0, 0.0, 0.0]),
+                    np.array([0.0, 1.0, 0.0]),
+                    np.array([1.0, 1.0, 1.0]),
+                ]
+        for vertex in vertices:
+            if np.dot(vertex - O, N) != 0:  # vertex not in plane
+                new_origin = vertex
+                break
+        # obtain new axes with center in new origin
+        X = np.array([1 - new_origin[0], new_origin[1], new_origin[2]])
+        Y = np.array([new_origin[0], 1 - new_origin[1], new_origin[2]])
+        Z = np.array([new_origin[0], new_origin[1], 1 - new_origin[2]])
+        new_Ccs = [X - new_origin, Y - new_origin, Z - new_origin]
+        segments = ([np.dot(P - new_origin, N) / np.dot(ort, N) if
+                    np.dot(ort, N) != 0 else np.nan for ort in new_Ccs])
+        # fix signs of indices: 0 -> 1, 1 -> -1 (
+        segments = (1 - 2 * new_origin) * segments
+
+    return sub_miller(segments)
+
+
+def grade(user_input, correct_answer):
+    '''
+    Grade crystallography problem.
+
+    Returns true if lattices are the same and Miller indices are same or minus
+    same. E.g. (2,2,2) = (2, 2, 2) or (-2, -2, -2). Because sign depends only
+    on student's selection of origin.
+
+    Args:
+        user_input, correct_answer: json. Format:
+
+        user_input: {"lattice":"sc","points":[["0.77","0.00","1.00"],
+        ["0.78","1.00","0.00"],["0.00","1.00","0.72"]]}
+
+        correct_answer: {'miller': '(00-1)', 'lattice': 'bcc'}
+
+        "lattice" is one of: "", "sc", "bcc", "fcc"
+
+    Returns:
+        True or false.
+    '''
+    def negative(m):
+        """
+        Change sign of Miller indices.
+
+        Args:
+            m: string with meaning of Miller indices. E.g.:
+            (-6,3,-6) -> (6, -3, 6)
+
+        Returns:
+            String with changed signs.
+        """
+        output = ''
+        i = 1
+        while i in range(1, len(m) - 1):
+            if m[i] in (',', ' '):
+                output += m[i]
+            elif m[i] not in ('-', '0'):
+                output += '-' + m[i]
+            elif m[i] == '0':
+                output += m[i]
+            else:
+                i += 1
+                output += m[i]
+            i += 1
+        return '(' + output + ')'
+
+    def round0_25(point):
+        """
+        Rounds point coordinates to closest 0.5 value.
+
+        Args:
+            point: list of float coordinates. Order of coordinates: x, y, z.
+
+        Returns:
+            list of coordinates rounded to closes 0.5 value
+        """
+        rounded_points = []
+        for coord in point:
+            base = math.floor(coord * 10)
+            fractional_part = (coord * 10 - base)
+            aliquot0_25 = math.floor(fractional_part / 0.25)
+            if aliquot0_25 == 0.0:
+                rounded_points.append(base / 10)
+            if aliquot0_25 in (1.0, 2.0):
+                rounded_points.append(base / 10 + 0.05)
+            if aliquot0_25 == 3.0:
+                rounded_points.append(base / 10 + 0.1)
+        return rounded_points
+
+    user_answer = json.loads(user_input)
+
+    if user_answer['lattice'] != correct_answer['lattice']:
+        return False
+
+    points = [map(float, p) for p in user_answer['points']]
+
+    if len(points) < 3:
+        return False
+
+    # round point to closes 0.05 value
+    points = [round0_25(point) for point in points]
+
+    points = [np.array(point) for point in points]
+    # print miller(points), (correct_answer['miller'].replace(' ', ''),
+    #     negative(correct_answer['miller']).replace(' ', ''))
+    if miller(points) in (correct_answer['miller'].replace(' ', ''), negative(correct_answer['miller']).replace(' ', '')):
+        return True
+
+    return False

common/lib/capa/capa/chem/tests.py

@@ -1,13 +1,15 @@
 import codecs
 from fractions import Fraction
-from pyparsing import ParseException
 import unittest
 
 from chemcalc import (compare_chemical_expression, divide_chemical_expression,
                       render_to_html, chemical_equations_equal)
 
+import miller
+
 local_debug = None
 
+
 def log(s, output_type=None):
     if local_debug:
         print s
@@ -37,7 +39,6 @@ class Test_Compare_Equations(unittest.TestCase):
         self.assertFalse(chemical_equations_equal('2H2 + O2 -> H2O2',
                                                  '2O2 + 2H2 -> 2H2O2'))
 
-
     def test_different_arrows(self):
         self.assertTrue(chemical_equations_equal('H2 + O2 -> H2O2',
                                                  '2O2 + 2H2 -> 2H2O2'))
@@ -56,7 +57,6 @@ class Test_Compare_Equations(unittest.TestCase):
         self.assertTrue(chemical_equations_equal('H2 + O2 -> H2O2',
                                                  'O2 + H2 -> H2O2', exact=True))
 
-
     def test_syntax_errors(self):
         self.assertFalse(chemical_equations_equal('H2 + O2 a-> H2O2',
                                                   '2O2 + 2H2 -> 2H2O2'))
@@ -311,7 +311,6 @@ class Test_Render_Equations(unittest.TestCase):
         log(out + ' ------- ' + correct, 'html')
         self.assertEqual(out, correct)
 
-        
     def test_render_eq3(self):
         s = "H^+ + OH^- <= H2O"   # unsupported arrow
         out = render_to_html(s)
@@ -320,10 +319,148 @@ class Test_Render_Equations(unittest.TestCase):
         self.assertEqual(out, correct)
 
 
+class Test_Crystallography_Miller(unittest.TestCase):
+    ''' Tests  for crystallography grade function.'''
+
+    def test_empty_points(self):
+        user_input = '{"lattice": "bcc", "points": []}'
+        self.assertFalse(miller.grade(user_input, {'miller': '(2,2,2)', 'lattice': 'bcc'}))
+
+    def test_only_one_point(self):
+        user_input = '{"lattice": "bcc", "points": [["0.50", "0.00", "0.00"]]}'
+        self.assertFalse(miller.grade(user_input, {'miller': '(2,2,2)', 'lattice': 'bcc'}))
+
+    def test_only_two_points(self):
+        user_input = '{"lattice": "bcc", "points": [["0.50", "0.00", "0.00"], ["0.00", "0.50", "0.00"]]}'
+        self.assertFalse(miller.grade(user_input, {'miller': '(2,2,2)', 'lattice': 'bcc'}))
+
+    def test_1(self):
+        user_input = '{"lattice": "bcc", "points": [["0.50", "0.00", "0.00"], ["0.00", "0.50", "0.00"], ["0.00", "0.00", "0.50"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(2,2,2)', 'lattice': 'bcc'}))
+
+    def test_2(self):
+        user_input = '{"lattice": "bcc", "points": [["1.00", "0.00", "0.00"], ["0.00", "1.00", "0.00"], ["0.00", "0.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(1,1,1)', 'lattice': 'bcc'}))
+
+    def test_3(self):
+        user_input = '{"lattice": "bcc", "points": [["1.00", "0.50", "1.00"], ["1.00", "1.00", "0.50"], ["0.50", "1.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(2,2,2)', 'lattice': 'bcc'}))
+
+    def test_4(self):
+        user_input = '{"lattice": "bcc", "points": [["0.33", "1.00", "0.00"], ["0.00", "0.664", "0.00"], ["0.00", "1.00", "0.33"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(-3, 3, -3)', 'lattice': 'bcc'}))
+
+    def test_5(self):
+        """ return true only in case points coordinates are exact.
+        But if they transform to closest 0.05 value it is not true"""
+        user_input = '{"lattice": "bcc", "points": [["0.33", "1.00", "0.00"], ["0.00", "0.33", "0.00"], ["0.00", "1.00", "0.33"]]}'
+        self.assertFalse(miller.grade(user_input, {'miller': '(-6,3,-6)', 'lattice': 'bcc'}))
+
+    def test_6(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.25", "0.00"], ["0.25", "0.00", "0.00"], ["0.00", "0.00", "0.25"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(4,4,4)', 'lattice': 'bcc'}))
+
+    def test_7(self):  # goes throug origin
+        user_input = '{"lattice": "bcc", "points": [["0.00", "1.00", "0.00"], ["1.00", "0.00", "0.00"], ["0.50", "1.00", "0.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(0,0,-1)', 'lattice': 'bcc'}))
+
+    def test_8(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "1.00", "0.50"], ["1.00", "0.00", "0.50"], ["0.50", "1.00", "0.50"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(0,0,2)', 'lattice': 'bcc'}))
+
+    def test_9(self):
+        user_input = '{"lattice": "bcc", "points": [["1.00", "0.00", "1.00"], ["0.00", "1.00", "1.00"], ["1.00", "0.00", "0.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(1,1,0)', 'lattice': 'bcc'}))
+
+    def test_10(self):
+        user_input = '{"lattice": "bcc", "points": [["1.00", "0.00", "1.00"], ["0.00", "0.00", "0.00"], ["0.00", "1.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(1,1,-1)', 'lattice': 'bcc'}))
+
+    def test_11(self):
+        user_input = '{"lattice": "bcc", "points": [["1.00", "0.00", "0.50"], ["1.00", "1.00", "0.00"], ["0.00", "1.00", "0.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(0,1,2)', 'lattice': 'bcc'}))
+
+    def test_12(self):
+        user_input = '{"lattice": "bcc", "points": [["1.00", "0.00", "0.50"], ["0.00", "0.00", "0.50"], ["1.00", "1.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(0,1,-2)', 'lattice': 'bcc'}))
+
+    def test_13(self):
+        user_input = '{"lattice": "bcc", "points": [["0.50", "0.00", "0.00"], ["0.50", "1.00", "0.00"], ["0.00", "0.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(2,0,1)', 'lattice': 'bcc'}))
+
+    def test_14(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.00", "0.00"], ["0.00", "0.00", "1.00"], ["0.50", "1.00", "0.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(2,-1,0)', 'lattice': 'bcc'}))
+
+    def test_15(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.00", "0.00"], ["1.00", "1.00", "0.00"], ["0.00", "1.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(1,-1,1)', 'lattice': 'bcc'}))
+
+    def test_16(self):
+        user_input = '{"lattice": "bcc", "points": [["1.00", "0.00", "0.00"], ["0.00", "1.00", "0.00"], ["1.00", "1.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(1,1,-1)', 'lattice': 'bcc'}))
+
+    def test_17(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.00", "0.00"], ["1.00", "0.00", "1.00"], ["1.00", "1.00", "0.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(-1,1,1)', 'lattice': 'bcc'}))
+
+    def test_18(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.00", "0.00"], ["1.00", "1.00", "0.00"], ["0.00", "1.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(1,-1,1)', 'lattice': 'bcc'}))
+
+    def test_19(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.00", "0.00"], ["1.00", "1.00", "0.00"], ["0.00", "0.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(-1,1,0)', 'lattice': 'bcc'}))
+
+    def test_20(self):
+        user_input = '{"lattice": "bcc", "points": [["1.00", "0.00", "0.00"], ["1.00", "1.00", "0.00"], ["0.00", "0.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(1,0,1)', 'lattice': 'bcc'}))
+
+    def test_21(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.00", "0.00"], ["0.00", "1.00", "0.00"], ["1.00", "0.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(-1,0,1)', 'lattice': 'bcc'}))
+
+    def test_22(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "1.00", "0.00"], ["1.00", "1.00", "0.00"], ["0.00", "0.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(0,1,1)', 'lattice': 'bcc'}))
+
+    def test_23(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.00", "0.00"], ["1.00", "0.00", "0.00"], ["1.00", "1.00", "1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(0,-1,1)', 'lattice': 'bcc'}))
+
+    def test_24(self):
+        user_input = '{"lattice": "bcc", "points": [["0.66", "0.00", "0.00"], ["0.00", "0.66", "0.00"], ["0.00", "0.00", "0.66"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(3,3,3)', 'lattice': 'bcc'}))
+
+    def test_25(self):
+        user_input = u'{"lattice":"","points":[["0.00","0.00","0.01"],["1.00","1.00","0.01"],["0.00","1.00","1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(1,-1,1)', 'lattice': ''}))
+
+    def test_26(self):
+        user_input = u'{"lattice":"","points":[["0.00","0.01","0.00"],["1.00","0.00","0.00"],["0.00","0.00","1.00"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(0,-1,0)', 'lattice': ''}))
+
+    def test_27(self):
+        """ rounding to 0.35"""
+        user_input = u'{"lattice":"","points":[["0.33","0.00","0.00"],["0.00","0.33","0.00"],["0.00","0.00","0.33"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(3,3,3)', 'lattice': ''}))
+
+    def test_28(self):
+        """ rounding to 0.30"""
+        user_input = u'{"lattice":"","points":[["0.30","0.00","0.00"],["0.00","0.30","0.00"],["0.00","0.00","0.30"]]}'
+        self.assertTrue(miller.grade(user_input, {'miller': '(10,10,10)', 'lattice': ''}))
+
+    def test_wrong_lattice(self):
+        user_input = '{"lattice": "bcc", "points": [["0.00", "0.00", "0.00"], ["1.00", "0.00", "0.00"], ["1.00", "1.00", "1.00"]]}'
+        self.assertFalse(miller.grade(user_input, {'miller': '(3,3,3)', 'lattice': 'fcc'}))
+
 
 def suite():
 
-    testcases = [Test_Compare_Expressions, Test_Divide_Expressions, Test_Render_Equations]
+    testcases = [Test_Compare_Expressions,
+                 Test_Divide_Expressions,
+                 Test_Render_Equations,
+                 Test_Crystallography_Miller]
     suites = []
     for testcase in testcases:
         suites.append(unittest.TestLoader().loadTestsFromTestCase(testcase))

common/lib/capa/capa/inputtypes.py

@@ -671,18 +671,15 @@ class Crystallography(InputTypeBase):
         """
         Note: height, width are required.
         """
-        return [Attribute('size', None),
-                Attribute('height'),
+        return [Attribute('height'),
                 Attribute('width'),
-
-                # can probably be removed (textline should prob be always-hidden)
-                Attribute('hidden', ''),
                 ]
 
 registry.register(Crystallography)
 
 # -------------------------------------------------------------------------
 
+
 class VseprInput(InputTypeBase):
     """
     Input for molecular geometry--show possible structures, let student
@@ -736,3 +733,53 @@ class ChemicalEquationInput(InputTypeBase):
         return {'previewer': '/static/js/capa/chemical_equation_preview.js',}
 
 registry.register(ChemicalEquationInput)
+
+#-----------------------------------------------------------------------------
+
+class OpenEndedInput(InputTypeBase):
+    """
+    A text area input for code--uses codemirror, does syntax highlighting, special tab handling,
+    etc.
+    """
+
+    template = "openendedinput.html"
+    tags = ['openendedinput']
+
+    # pulled out for testing
+    submitted_msg = ("Feedback not yet available.  Reload to check again. "
+                     "Once the problem is graded, this message will be "
+                     "replaced with the grader's feedback")
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Convert options to a convenient format.
+        """
+        return [Attribute('rows', '30'),
+                Attribute('cols', '80'),
+                Attribute('hidden', ''),
+                ]
+
+    def setup(self):
+        """
+        Implement special logic: handle queueing state, and default input.
+        """
+        # if no student input yet, then use the default input given by the problem
+        if not self.value:
+            self.value = self.xml.text
+
+        # Check if problem has been queued
+        self.queue_len = 0
+        # Flag indicating that the problem has been queued, 'msg' is length of queue
+        if self.status == 'incomplete':
+            self.status = 'queued'
+            self.queue_len = self.msg
+            self.msg = self.submitted_msg
+
+    def _extra_context(self):
+        """Defined queue_len, add it """
+        return {'queue_len': self.queue_len,}
+
+registry.register(OpenEndedInput)
+
+#-----------------------------------------------------------------------------

common/lib/capa/capa/responsetypes.py

@@ -8,22 +8,25 @@ Used by capa_problem.py
 '''
 
 # standard library imports
+import abc
 import cgi
+import hashlib
 import inspect
 import json
 import logging
 import numbers
 import numpy
+import os
 import random
 import re
 import requests
-import traceback
-import hashlib
-import abc
-import os
 import subprocess
+import traceback
 import xml.sax.saxutils as saxutils
 
+from collections import namedtuple
+from shapely.geometry import Point, MultiPoint
+
 # specific library imports
 from calc import evaluator, UndefinedVariable
 from correctmap import CorrectMap
@@ -1104,6 +1107,15 @@ class SymbolicResponse(CustomResponse):
 
 #-----------------------------------------------------------------------------
 
+"""
+valid:       Flag indicating valid score_msg format (Boolean)
+correct:     Correctness of submission (Boolean)
+score:       Points to be assigned (numeric, can be float)
+msg:         Message from grader to display to student (string)
+"""
+ScoreMessage = namedtuple('ScoreMessage',
+                          ['valid', 'correct', 'points', 'msg'])
+
 
 class CodeResponse(LoncapaResponse):
     """
@@ -1149,7 +1161,7 @@ class CodeResponse(LoncapaResponse):
         else:
             self._parse_coderesponse_xml(codeparam)
 
-    def _parse_coderesponse_xml(self,codeparam):
+    def _parse_coderesponse_xml(self, codeparam):
         '''
         Parse the new CodeResponse XML format. When successful, sets:
             self.initial_display
@@ -1161,17 +1173,9 @@ class CodeResponse(LoncapaResponse):
         grader_payload = grader_payload.text if grader_payload is not None else ''
         self.payload = {'grader_payload': grader_payload}
 
-        answer_display = codeparam.find('answer_display')
-        if answer_display is not None:
-            self.answer = answer_display.text
-        else:
-            self.answer = 'No answer provided.'
-
-        initial_display = codeparam.find('initial_display')
-        if initial_display is not None:
-            self.initial_display = initial_display.text
-        else:
-            self.initial_display = ''
+        self.initial_display = find_with_default(codeparam, 'initial_display', '')
+        self.answer = find_with_default(codeparam, 'answer_display',
+                                        'No answer provided.')
 
     def _parse_externalresponse_xml(self):
         '''
@@ -1325,8 +1329,6 @@ class CodeResponse(LoncapaResponse):
             # Sanity check on returned points
             if points < 0:
                 points = 0
-            elif points > self.maxpoints[self.answer_id]:
-                points = self.maxpoints[self.answer_id]
             # Queuestate is consumed
             oldcmap.set(self.answer_id, npoints=points, correctness=correctness,
                         msg=msg.replace('&nbsp;', '&#160;'), queuestate=None)
@@ -1734,15 +1736,38 @@ class ImageResponse(LoncapaResponse):
     which produces an [x,y] coordinate pair.  The click is correct if it falls
     within a region specified.  This region is a union of rectangles.
 
-    Lon-CAPA requires that each <imageresponse> has a <foilgroup> inside it.  That
-    doesn't make sense to me (Ike).  Instead, let's have it such that <imageresponse>
-    should contain one or more <imageinput> stanzas. Each <imageinput> should specify
-    a rectangle, given as an attribute, defining the correct answer.
+    Lon-CAPA requires that each <imageresponse> has a <foilgroup> inside it.
+    That doesn't make sense to me (Ike).  Instead, let's have it such that
+    <imageresponse> should contain one or more <imageinput> stanzas.
+    Each <imageinput> should specify a rectangle(s) or region(s), given as an
+    attribute, defining the correct answer.
+
+    <imageinput src="/static/images/Lecture2/S2_p04.png" width="811" height="610"
+    rectangle="(10,10)-(20,30);(12,12)-(40,60)"
+    regions="[[[10,10], [20,30], [40, 10]], [[100,100], [120,130], [110,150]]]"/>
+
+    Regions is list of lists [region1, region2, region3, ...] where regionN
+    is disordered list of points: [[1,1], [100,100], [50,50], [20, 70]].
+
+    If there is only one region in the list, simpler notation can be used:
+    regions="[[10,10], [30,30], [10, 30], [30, 10]]" (without explicitly
+        setting outer list)
+
+    Returns:
+        True, if click is inside any region or rectangle. Otherwise False.
     """
     snippets = [{'snippet': '''<imageresponse>
-      <imageinput src="image1.jpg" width="200" height="100" rectangle="(10,10)-(20,30)" />
-      <imageinput src="image2.jpg" width="210" height="130" rectangle="(12,12)-(40,60)" />
-      <imageinput src="image2.jpg" width="210" height="130" rectangle="(10,10)-(20,30);(12,12)-(40,60)" />
+      <imageinput src="image1.jpg" width="200" height="100"
+      rectangle="(10,10)-(20,30)" />
+      <imageinput src="image2.jpg" width="210" height="130"
+      rectangle="(12,12)-(40,60)" />
+      <imageinput src="image3.jpg" width="210" height="130"
+      rectangle="(10,10)-(20,30);(12,12)-(40,60)" />
+      <imageinput src="image4.jpg" width="811" height="610"
+      rectangle="(10,10)-(20,30);(12,12)-(40,60)"
+      regions="[[[10,10], [20,30], [40, 10]], [[100,100], [120,130], [110,150]]]"/>
+      <imageinput src="image5.jpg" width="200" height="200"
+      regions="[[[10,10], [20,30], [40, 10]], [[100,100], [120,130], [110,150]]]"/>
     </imageresponse>'''}]
 
     response_tag = 'imageresponse'
@@ -1750,19 +1775,17 @@ class ImageResponse(LoncapaResponse):
 
     def setup_response(self):
         self.ielements = self.inputfields
-        self.answer_ids = [ie.get('id')  for ie in self.ielements]
+        self.answer_ids = [ie.get('id') for ie in self.ielements]
 
     def get_score(self, student_answers):
         correct_map = CorrectMap()
         expectedset = self.get_answers()
-
-        for aid in self.answer_ids:	 # loop through IDs of <imageinput> fields in our stanza
-            given = student_answers[aid]	 # this should be a string of the form '[x,y]'
-
+        for aid in self.answer_ids:	 # loop through IDs of <imageinput>
+        #  fields in our stanza
+            given = student_answers[aid]  # this should be a string of the form '[x,y]'
             correct_map.set(aid, 'incorrect')
-            if not given: # No answer to parse. Mark as incorrect and move on
+            if not given:  # No answer to parse. Mark as incorrect and move on
                 continue
-
             # parse given answer
             m = re.match('\[([0-9]+),([0-9]+)]', given.strip().replace(' ', ''))
             if not m:
@@ -1770,28 +1793,384 @@ class ImageResponse(LoncapaResponse):
                                 'error grading %s (input=%s)' % (aid, given))
             (gx, gy) = [int(x) for x in m.groups()]
 
-            # Check whether given point lies in any of the solution rectangles
-            solution_rectangles = expectedset[aid].split(';')
-            for solution_rectangle in solution_rectangles:
-                # parse expected answer
-                # TODO: Compile regexp on file load
-                m = re.match('[\(\[]([0-9]+),([0-9]+)[\)\]]-[\(\[]([0-9]+),([0-9]+)[\)\]]',
-                             solution_rectangle.strip().replace(' ', ''))
-                if not m:
-                    msg = 'Error in problem specification! cannot parse rectangle in %s' % (
-                        etree.tostring(self.ielements[aid], pretty_print=True))
-                    raise Exception('[capamodule.capa.responsetypes.imageinput] ' + msg)
-                (llx, lly, urx, ury) = [int(x) for x in m.groups()]
-
-                # answer is correct if (x,y) is within the specified rectangle
-                if (llx <= gx <= urx) and (lly <= gy <= ury):
-                    correct_map.set(aid, 'correct')
-                    break
+            rectangles, regions = expectedset
+            if rectangles[aid]:  # rectangles part - for backward compatibility
+                # Check whether given point lies in any of the solution rectangles
+                solution_rectangles = rectangles[aid].split(';')
+                for solution_rectangle in solution_rectangles:
+                    # parse expected answer
+                    # TODO: Compile regexp on file load
+                    m = re.match('[\(\[]([0-9]+),([0-9]+)[\)\]]-[\(\[]([0-9]+),([0-9]+)[\)\]]',
+                                 solution_rectangle.strip().replace(' ', ''))
+                    if not m:
+                        msg = 'Error in problem specification! cannot parse rectangle in %s' % (
+                            etree.tostring(self.ielements[aid], pretty_print=True))
+                        raise Exception('[capamodule.capa.responsetypes.imageinput] ' + msg)
+                    (llx, lly, urx, ury) = [int(x) for x in m.groups()]
 
+                    # answer is correct if (x,y) is within the specified rectangle
+                    if (llx <= gx <= urx) and (lly <= gy <= ury):
+                        correct_map.set(aid, 'correct')
+                        break
+            if correct_map[aid]['correctness'] != 'correct' and regions[aid]:
+                parsed_region = json.loads(regions[aid])
+                if parsed_region:
+                    if type(parsed_region[0][0]) != list:
+                        # we have [[1,2],[3,4],[5,6]] - single region
+                        # instead of [[[1,2],[3,4],[5,6], [[1,2],[3,4],[5,6]]]
+                        # or [[[1,2],[3,4],[5,6]]] - multiple regions syntax
+                        parsed_region = [parsed_region]
+                    for region in parsed_region:
+                        polygon = MultiPoint(region).convex_hull
+                        if (polygon.type == 'Polygon' and
+                                polygon.contains(Point(gx, gy))):
+                            correct_map.set(aid, 'correct')
+                            break
         return correct_map
 
     def get_answers(self):
-        return dict([(ie.get('id'), ie.get('rectangle')) for ie in self.ielements])
+        return (dict([(ie.get('id'), ie.get('rectangle')) for ie in self.ielements]),
+                dict([(ie.get('id'), ie.get('regions')) for ie in self.ielements]))
+#-----------------------------------------------------------------------------
+
+class OpenEndedResponse(LoncapaResponse):
+    """
+    Grade student open ended responses using an external grading system,
+    accessed through the xqueue system.
+
+    Expects 'xqueue' dict in ModuleSystem with the following keys that are
+    needed by OpenEndedResponse:
+
+        system.xqueue = { 'interface': XqueueInterface object,
+                          'callback_url': Per-StudentModule callback URL
+                                          where results are posted (string),
+                        }
+
+    External requests are only submitted for student submission grading
+        (i.e. and not for getting reference answers)
+
+    By default, uses the OpenEndedResponse.DEFAULT_QUEUE queue.
+    """
+
+    DEFAULT_QUEUE = 'open-ended'
+    response_tag = 'openendedresponse'
+    allowed_inputfields = ['openendedinput']
+    max_inputfields = 1
+
+    def setup_response(self):
+        '''
+        Configure OpenEndedResponse from XML.
+        '''
+        xml = self.xml
+        self.url = xml.get('url', None)
+        self.queue_name = xml.get('queuename', self.DEFAULT_QUEUE)
+
+        # The openendedparam tag encapsulates all grader settings
+        oeparam = self.xml.find('openendedparam')
+        prompt = self.xml.find('prompt')
+        rubric = self.xml.find('openendedrubric')
+
+        if oeparam is None:
+            raise ValueError("No oeparam found in problem xml.")
+        if prompt is None:
+            raise ValueError("No prompt found in problem xml.")
+        if rubric is None:
+            raise ValueError("No rubric found in problem xml.")
+
+        self._parse(oeparam, prompt, rubric)
+
+    @staticmethod
+    def stringify_children(node):
+        """
+        Modify code from stringify_children in xmodule.  Didn't import directly
+        in order to avoid capa depending on xmodule (seems to be avoided in
+        code)
+        """
+        parts=[node.text if node.text is not None else '']
+        for p in node.getchildren():
+            parts.append(etree.tostring(p, with_tail=True, encoding='unicode'))
+
+        return ' '.join(parts)
+
+    def _parse(self, oeparam, prompt, rubric):
+        '''
+        Parse OpenEndedResponse XML:
+            self.initial_display
+            self.payload - dict containing keys --
+            'grader' : path to grader settings file, 'problem_id' : id of the problem
+
+            self.answer - What to display when show answer is clicked
+        '''
+        # Note that OpenEndedResponse is agnostic to the specific contents of grader_payload
+        prompt_string = self.stringify_children(prompt)
+        rubric_string = self.stringify_children(rubric)
+
+        grader_payload = oeparam.find('grader_payload')
+        grader_payload = grader_payload.text if grader_payload is not None else ''
+
+        #Update grader payload with student id.  If grader payload not json, error.
+        try:
+            parsed_grader_payload = json.loads(grader_payload)
+            # NOTE: self.system.location is valid because the capa_module
+            # __init__ adds it (easiest way to get problem location into
+            # response types)
+        except TypeError, ValueError:
+            log.exception("Grader payload %r is not a json object!", grader_payload)
+        parsed_grader_payload.update({
+            'location' : self.system.location,
+            'course_id' : self.system.course_id,
+            'prompt' : prompt_string,
+            'rubric' : rubric_string,
+        })
+        updated_grader_payload = json.dumps(parsed_grader_payload)
+
+        self.payload = {'grader_payload': updated_grader_payload}
+
+        self.initial_display = find_with_default(oeparam, 'initial_display', '')
+        self.answer = find_with_default(oeparam, 'answer_display', 'No answer given.')
+        try:
+            self.max_score = int(find_with_default(oeparam, 'max_score', 1))
+        except ValueError:
+            self.max_score = 1
+
+    def get_score(self, student_answers):
+
+        try:
+            submission = student_answers[self.answer_id]
+        except KeyError:
+            msg = ('Cannot get student answer for answer_id: {0}. student_answers {1}'
+                   .format(self.answer_id, student_answers))
+            log.exception(msg)
+            raise LoncapaProblemError(msg)
+
+        # Prepare xqueue request
+        #------------------------------------------------------------
+
+        qinterface = self.system.xqueue['interface']
+        qtime = datetime.strftime(datetime.now(), xqueue_interface.dateformat)
+
+        anonymous_student_id = self.system.anonymous_student_id
+
+        # Generate header
+        queuekey = xqueue_interface.make_hashkey(str(self.system.seed) + qtime +
+                                                 anonymous_student_id +
+                                                 self.answer_id)
+
+        xheader = xqueue_interface.make_xheader(lms_callback_url=self.system.xqueue['callback_url'],
+            lms_key=queuekey,
+            queue_name=self.queue_name)
+
+        self.context.update({'submission': submission})
+
+        contents = self.payload.copy()
+
+        # Metadata related to the student submission revealed to the external grader
+        student_info = {'anonymous_student_id': anonymous_student_id,
+                        'submission_time': qtime,
+                        }
+
+        #Update contents with student response and student info
+        contents.update({
+            'student_info': json.dumps(student_info),
+            'student_response': submission,
+            'max_score' : self.max_score
+            })
+
+        # Submit request. When successful, 'msg' is the prior length of the queue
+        (error, msg) = qinterface.send_to_queue(header=xheader,
+            body=json.dumps(contents))
+
+        # State associated with the queueing request
+        queuestate = {'key': queuekey,
+                      'time': qtime,}
+
+        cmap = CorrectMap()
+        if error:
+            cmap.set(self.answer_id, queuestate=None,
+                msg='Unable to deliver your submission to grader. (Reason: {0}.)'
+                    ' Please try again later.'.format(msg))
+        else:
+            # Queueing mechanism flags:
+            #   1) Backend: Non-null CorrectMap['queuestate'] indicates that
+            #      the problem has been queued
+            #   2) Frontend: correctness='incomplete' eventually trickles down
+            #      through inputtypes.textbox and .filesubmission to inform the
+            #      browser that the submission is queued (and it could e.g. poll)
+            cmap.set(self.answer_id, queuestate=queuestate,
+                     correctness='incomplete', msg=msg)
+
+        return cmap
+
+    def update_score(self, score_msg, oldcmap, queuekey):
+        log.debug(score_msg)
+        score_msg = self._parse_score_msg(score_msg)
+        if not score_msg.valid:
+            oldcmap.set(self.answer_id,
+                msg = 'Invalid grader reply. Please contact the course staff.')
+            return oldcmap
+
+        correctness = 'correct' if score_msg.correct else 'incorrect'
+
+        # TODO: Find out how this is used elsewhere, if any
+        self.context['correct'] = correctness
+
+        # Replace 'oldcmap' with new grading results if queuekey matches.  If queuekey
+        # does not match, we keep waiting for the score_msg whose key actually matches
+        if oldcmap.is_right_queuekey(self.answer_id, queuekey):
+            # Sanity check on returned points
+            points = score_msg.points
+            if points < 0:
+                points = 0
+
+            # Queuestate is consumed, so reset it to None
+            oldcmap.set(self.answer_id, npoints=points, correctness=correctness,
+                msg = score_msg.msg.replace('&nbsp;', '&#160;'), queuestate=None)
+        else:
+            log.debug('OpenEndedResponse: queuekey {0} does not match for answer_id={1}.'.format(
+                queuekey, self.answer_id))
+
+        return oldcmap
+
+    def get_answers(self):
+        anshtml = '<span class="openended-answer"><pre><code>{0}</code></pre></span>'.format(self.answer)
+        return {self.answer_id: anshtml}
+
+    def get_initial_display(self):
+        return {self.answer_id: self.initial_display}
+
+    def _convert_longform_feedback_to_html(self, response_items):
+        """
+        Take in a dictionary, and return html strings for display to student.
+        Input:
+            response_items: Dictionary with keys success, feedback.
+                if success is True, feedback should be a dictionary, with keys for
+                   types of feedback, and the corresponding feedback values.
+                if success is False, feedback is actually an error string.
+
+                NOTE: this will need to change when we integrate peer grading, because
+                that will have more complex feedback.
+
+        Output:
+            String -- html that can be displayed to the student.
+        """
+
+        # We want to display available feedback in a particular order.
+        # This dictionary specifies which goes first--lower first.
+        priorities = {# These go at the start of the feedback
+                      'spelling': 0,
+                      'grammar': 1,
+                      # needs to be after all the other feedback
+                      'markup_text': 3}
+
+        default_priority = 2
+
+        def get_priority(elt):
+            """
+            Args:
+                elt: a tuple of feedback-type, feedback
+            Returns:
+                the priority for this feedback type
+            """
+            return priorities.get(elt[0], default_priority)
+
+        def format_feedback(feedback_type, value):
+            return """
+            <div class="{feedback_type}">
+            {value}
+            </div>
+            """.format(feedback_type=feedback_type, value=value)
+
+        # TODO (vshnayder): design and document the details of this format so
+        # that we can do proper escaping here (e.g. are the graders allowed to
+        # include HTML?)
+
+        for tag in ['success', 'feedback']:
+            if tag not in response_items:
+                return format_feedback('errors', 'Error getting feedback')
+
+        feedback_items = response_items['feedback']
+        try:
+            feedback = json.loads(feedback_items)
+        except (TypeError, ValueError):
+            log.exception("feedback_items have invalid json %r", feedback_items)
+            return format_feedback('errors', 'Could not parse feedback')
+
+        if response_items['success']:
+            if len(feedback) == 0:
+                return format_feedback('errors', 'No feedback available')
+
+            feedback_lst = sorted(feedback.items(), key=get_priority)
+            return u"\n".join(format_feedback(k, v) for k, v in feedback_lst)
+        else:
+            return format_feedback('errors', response_items['feedback'])
+
+
+    def _format_feedback(self, response_items):
+        """
+        Input:
+            Dictionary called feedback.  Must contain keys seen below.
+        Output:
+            Return error message or feedback template
+        """
+
+        feedback = self._convert_longform_feedback_to_html(response_items)
+
+        if not response_items['success']:
+            return self.system.render_template("open_ended_error.html",
+                                               {'errors' : feedback})
+
+        feedback_template = self.system.render_template("open_ended_feedback.html", {
+            'grader_type': response_items['grader_type'],
+            'score': response_items['score'],
+            'feedback': feedback,
+        })
+
+        return feedback_template
+
+
+    def _parse_score_msg(self, score_msg):
+        """
+         Grader reply is a JSON-dump of the following dict
+           { 'correct': True/False,
+             'score': Numeric value (floating point is okay) to assign to answer
+             'msg': grader_msg
+             'feedback' : feedback from grader
+             }
+
+        Returns (valid_score_msg, correct, score, msg):
+            valid_score_msg: Flag indicating valid score_msg format (Boolean)
+            correct:         Correctness of submission (Boolean)
+            score:           Points to be assigned (numeric, can be float)
+        """
+        fail = ScoreMessage(valid=False, correct=False, points=0, msg='')
+        try:
+            score_result = json.loads(score_msg)
+        except (TypeError, ValueError):
+            log.error("External grader message should be a JSON-serialized dict."
+                      " Received score_msg = {0}".format(score_msg))
+            return fail
+
+        if not isinstance(score_result, dict):
+            log.error("External grader message should be a JSON-serialized dict."
+                      " Received score_result = {0}".format(score_result))
+            return fail
+
+        for tag in ['score', 'feedback', 'grader_type', 'success']:
+            if tag not in score_result:
+                log.error("External grader message is missing required tag: {0}"
+                          .format(tag))
+                return fail
+
+        feedback = self._format_feedback(score_result)
+
+        # HACK: for now, just assume it's correct if you got more than 2/3.
+        # Also assumes that score_result['score'] is an integer.
+        score_ratio = int(score_result['score']) / self.max_score
+        correct = (score_ratio >= 0.66)
+
+        #Currently ignore msg and only return feedback (which takes the place of msg)
+        return ScoreMessage(valid=True, correct=correct,
+                            points=score_result['score'], msg=feedback)
 
 #-----------------------------------------------------------------------------
 # TEMPORARY: List of all response subclasses
@@ -1810,4 +2189,5 @@ __all__ = [CodeResponse,
            ChoiceResponse,
            MultipleChoiceResponse,
            TrueFalseResponse,
-           JavascriptResponse]
+           JavascriptResponse,
+           OpenEndedResponse]

common/lib/capa/capa/templates/crystallography.html

@@ -1,34 +1,28 @@
 <section id="inputtype_${id}" class="capa_inputtype" >
-     <div id="holder" style="width:${width};height:${height}"></div>
+    <div class="crystalography_problem" style="width:${width};height:${height}"></div>
+
+    <div class="input_lattice">
+        Lattice: <select></select>
+    </div>
+
     <div class="script_placeholder" data-src="/static/js/raphael.js"></div>
     <div class="script_placeholder" data-src="/static/js/sylvester.js"></div>
-    <div class="script_placeholder" data-src="/static/js/underscore-min.js"></div>
     <div class="script_placeholder" data-src="/static/js/crystallography.js"></div>
 
-
     % if status == 'unsubmitted':
-    <div class="unanswered" id="status_${id}">
+        <div class="unanswered" id="status_${id}">
     % elif status == 'correct':
-    <div class="correct" id="status_${id}">
+        <div class="correct" id="status_${id}">
     % elif status == 'incorrect':
-    <div class="incorrect" id="status_${id}">
+        <div class="incorrect" id="status_${id}">
     % elif status == 'incomplete':
-    <div class="incorrect" id="status_${id}">
-    % endif
-    % if hidden:
-      <div style="display:none;" name="${hidden}" inputid="input_${id}" />
+        <div class="incorrect" id="status_${id}">
     % endif
 
-  <input type="text" name="input_${id}" id="input_${id}" value="${value|h}"
-   % if size:
-   size="${size}"
-   % endif
-   % if hidden:
-    style="display:none;"
-   % endif
-   />
 
-      <p class="status">
+    <input type="text" name="input_${id}" id="input_${id}" value="${value|h}" style="display:none;"/>
+
+    <p class="status">
         % if status == 'unsubmitted':
         unanswered
         % elif status == 'correct':
@@ -38,14 +32,15 @@
         % elif status == 'incomplete':
         incomplete
         % endif
-      </p>
+    </p>
 
-  <p id="answer_${id}" class="answer"></p>
+    <p id="answer_${id}" class="answer"></p>
 
-  % if msg:
-      <span class="message">${msg|n}</span>
-  % endif
-% if status in ['unsubmitted', 'correct', 'incorrect', 'incomplete']:
-</div>
-% endif
+    % if msg:
+        <span class="message">${msg|n}</span>
+    % endif
+
+    % if status in ['unsubmitted', 'correct', 'incorrect', 'incomplete']:
+        </div>
+    % endif
 </section>

common/lib/capa/capa/templates/openendedinput.html

@@ -0,0 +1,32 @@
+<section id="openended_${id}" class="openended">
+  <textarea rows="${rows}" cols="${cols}" name="input_${id}" class="short-form-response" id="input_${id}"
+       % if hidden:
+      	    style="display:none;"
+       % endif
+  >${value|h}</textarea>
+
+  <div class="grader-status">
+    % if status == 'unsubmitted':
+      <span class="unanswered" style="display:inline-block;" id="status_${id}">Unanswered</span>
+    % elif status == 'correct':
+      <span class="correct" id="status_${id}">Correct</span>
+    % elif status == 'incorrect':
+      <span class="incorrect" id="status_${id}">Incorrect</span>
+    % elif status == 'queued':
+      <span class="grading" id="status_${id}">Submitted for grading</span>
+    % endif
+
+    % if hidden:
+      <div style="display:none;" name="${hidden}" inputid="input_${id}" />
+    % endif
+  </div>
+
+  <span id="answer_${id}"></span>
+
+    % if status == 'queued':
+    <input name="reload" class="reload" type="button" value="Recheck for Feedback" onclick="document.location.reload(true);" />
+    % endif
+  <div class="external-grader-message">
+    ${msg|n}
+  </div>
+</section>

common/lib/capa/capa/tests/test_files/imageresponse.xml

@@ -18,4 +18,23 @@ Hello</p></text>
 <text><p>Use conservation of energy.</p></text>
 </hintgroup>
 </imageresponse>
+
+
+<imageresponse max="1" loncapaid="12">
+<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(490,11)-(556,98)" regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"/>
+<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(490,11)-(556,98)" regions='[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]'/>
+<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"/>
+<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"/>
+<text>Click on either of the two positions as discussed previously.</text>
+<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[[10,10], [20,10], [20, 30]]]"/>
+<text>Click on either of the two positions as discussed previously.</text>
+<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[10,10], [30,30], [15, 15]]"/>
+<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[10,10], [30,30], [10, 30], [30, 10]]"/>
+<text>Click on either of the two positions as discussed previously.</text>
+<hintgroup showoncorrect="no">
+<text><p>Use conservation of energy.</p></text>
+</hintgroup>
+</imageresponse>
+
+
 </problem>

common/lib/capa/capa/tests/test_inputtypes.py

@@ -407,13 +407,11 @@ class CrystallographyTest(unittest.TestCase):
     def test_rendering(self):
         height = '12'
         width = '33'
-        size = '10'
 
         xml_str = """<crystallography id="prob_1_2"
         height="{h}"
         width="{w}"
-        size="{s}"
-        />""".format(h=height, w=width, s=size)
+        />""".format(h=height, w=width)
 
         element = etree.fromstring(xml_str)
 
@@ -428,9 +426,7 @@ class CrystallographyTest(unittest.TestCase):
         expected = {'id': 'prob_1_2',
                     'value': value,
                     'status': 'unsubmitted',
-                    'size': size,
                     'msg': '',
-                    'hidden': '',
                     'width': width,
                     'height': height,
                    }

common/lib/capa/capa/tests/test_responsetypes.py

@@ -52,24 +52,57 @@ class ImageResponseTest(unittest.TestCase):
     def test_ir_grade(self):
         imageresponse_file = os.path.dirname(__file__) + "/test_files/imageresponse.xml"
         test_lcp = lcp.LoncapaProblem(open(imageresponse_file).read(), '1', system=test_system)
-        correct_answers = {'1_2_1': '(490,11)-(556,98)',
-                           '1_2_2': '(242,202)-(296,276)',
-                           '1_2_3': '(490,11)-(556,98);(242,202)-(296,276)',
-                           '1_2_4': '(490,11)-(556,98);(242,202)-(296,276)',
-                           '1_2_5': '(490,11)-(556,98);(242,202)-(296,276)',
+        # testing regions only
+        correct_answers = {
+           #regions
+           '1_2_1': '(490,11)-(556,98)',
+           '1_2_2': '(242,202)-(296,276)',
+           '1_2_3': '(490,11)-(556,98);(242,202)-(296,276)',
+           '1_2_4': '(490,11)-(556,98);(242,202)-(296,276)',
+           '1_2_5': '(490,11)-(556,98);(242,202)-(296,276)',
+           #testing regions and rectanges
+           '1_3_1': 'rectangle="(490,11)-(556,98)" \
+           regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"',
+           '1_3_2': 'rectangle="(490,11)-(556,98)" \
+           regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"',
+           '1_3_3': 'regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"',
+           '1_3_4': 'regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"',
+           '1_3_5': 'regions="[[[10,10], [20,10], [20, 30]]]"',
+           '1_3_6': 'regions="[[10,10], [30,30], [15, 15]]"',
+           '1_3_7': 'regions="[[10,10], [30,30], [10, 30], [30, 10]]"',
                           }
-        test_answers = {'1_2_1': '[500,20]',
-                        '1_2_2': '[250,300]',
-                        '1_2_3': '[500,20]',
-                        '1_2_4': '[250,250]',
-                        '1_2_5': '[10,10]',
+        test_answers = {
+            '1_2_1': '[500,20]',
+            '1_2_2': '[250,300]',
+            '1_2_3': '[500,20]',
+            '1_2_4': '[250,250]',
+            '1_2_5': '[10,10]',
+
+            '1_3_1': '[500,20]',
+            '1_3_2': '[15,15]',
+            '1_3_3': '[500,20]',
+            '1_3_4': '[115,115]',
+            '1_3_5': '[15,15]',
+            '1_3_6': '[20,20]',
+            '1_3_7': '[20,15]',
                         }
+
+        # regions
         self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_1'), 'correct')
         self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_2'), 'incorrect')
         self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_3'), 'correct')
         self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_4'), 'correct')
         self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_5'), 'incorrect')
 
+        # regions and rectangles
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_1'), 'correct')
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_2'), 'correct')
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_3'), 'incorrect')
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_4'), 'correct')
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_5'), 'correct')
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_6'), 'incorrect')
+        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_7'), 'correct')
+
 
 class SymbolicResponseTest(unittest.TestCase):
     def test_sr_grade(self):

common/lib/capa/capa/util.py

@@ -65,3 +65,25 @@ def is_file(file_to_test):
     Duck typing to check if 'file_to_test' is a File object
     '''
     return all(hasattr(file_to_test, method) for method in ['read', 'name'])
+
+
+def find_with_default(node, path, default):
+    """
+    Look for a child of node using , and return its text if found.
+    Otherwise returns default.
+
+    Arguments:
+       node: lxml node
+       path: xpath search expression
+       default: value to return if nothing found
+
+    Returns:
+       node.find(path).text if the find succeeds, default otherwise.
+       
+    """
+    v = node.find(path)
+    if v is not None:
+        return v.text
+    else:
+        return default
+

common/lib/capa/capa/xqueue_interface.py

@@ -49,6 +49,7 @@ def parse_xreply(xreply):
 
     return_code = xreply['return_code']
     content = xreply['content']
+
     return (return_code, content)
 
 
@@ -80,7 +81,11 @@ class XQueueInterface(object):
 
         # Log in, then try again
         if error and (msg == 'login_required'):
-            self._login()
+            (error, content) = self._login()
+            if error != 0:
+                # when the login fails
+                log.debug("Failed to login to queue: %s", content)
+                return (error, content)
             if files_to_upload is not None:
                 # Need to rewind file pointers
                 for f in files_to_upload:

common/lib/logsettings.py

@@ -45,7 +45,7 @@ def get_logger_config(log_dir,
                         logging_env=logging_env, hostname=hostname)
 
     handlers = ['console', 'local'] if debug else ['console',
-                                'syslogger-remote', 'local', 'newrelic']
+                                'syslogger-remote', 'local']
 
     logger_config = {
         'version': 1,

common/lib/xmodule/.coveragerc

@@ -7,6 +7,7 @@ source = common/lib/xmodule
 ignore_errors = True
 
 [html]
+title = XModule Python Test Coverage Report
 directory = reports/common/lib/xmodule/cover
 
 [xml]

common/lib/xmodule/xmodule/capa_module.py

@@ -145,6 +145,11 @@ class CapaModule(XModule):
         else:
             self.seed = None
 
+        # Need the problem location in openendedresponse to send out.  Adding
+        # it to the system here seems like the least clunky way to get it
+        # there.
+        self.system.set('location', self.location.url())
+
         try:
             # TODO (vshnayder): move as much as possible of this work and error
             # checking to descriptor load time

common/lib/xmodule/xmodule/course_module.py

@@ -186,7 +186,8 @@ class CourseDescriptor(SequenceDescriptor):
         instance = super(CourseDescriptor, cls).from_xml(xml_data, system, org, course)
 
         # bleh, have to parse the XML here to just pull out the url_name attribute
-        course_file = StringIO(xml_data)
+        # I don't think it's stored anywhere in the instance.
+        course_file = StringIO(xml_data.encode('ascii','ignore'))
         xml_obj = etree.parse(course_file,parser=edx_xml_parser).getroot()
 
         policy_dir = None
@@ -293,6 +294,10 @@ class CourseDescriptor(SequenceDescriptor):
         self.definition['data'].setdefault('grading_policy',{})['GRADE_CUTOFFS'] = value
     
 
+    @property
+    def lowest_passing_grade(self):
+        return min(self._grading_policy['GRADE_CUTOFFS'].values())
+
     @property
     def tabs(self):
         """
@@ -395,7 +400,20 @@ class CourseDescriptor(SequenceDescriptor):
 
     @property
     def start_date_text(self):
-        displayed_start = self._try_parse_time('advertised_start') or self.start
+        parsed_advertised_start = self._try_parse_time('advertised_start')
+
+        # If the advertised start isn't a real date string, we assume it's free
+        # form text...
+        if parsed_advertised_start is None and \
+           ('advertised_start' in self.metadata):
+           return self.metadata['advertised_start']
+
+        displayed_start = parsed_advertised_start or self.start
+
+        # If we have neither an advertised start or a real start, just return TBD
+        if not displayed_start:
+            return "TBD"
+
         return time.strftime("%b %d, %Y", displayed_start)
 
     @property
@@ -440,7 +458,7 @@ class CourseDescriptor(SequenceDescriptor):
                     return False
         except:
             log.exception("Error parsing discussion_blackouts for course {0}".format(self.id))
-        
+
         return True
 
     @property

common/lib/xmodule/xmodule/css/capa/display.scss

@@ -121,16 +121,6 @@ section.problem {
       }
     }
 
-    &.processing {
-      p.status {
-        @include inline-block();
-        background: url('../images/spinner.gif') center center no-repeat;
-        height: 20px;
-        width: 20px;
-        text-indent: -9999px;
-      }
-    }
-
     &.correct, &.ui-icon-check {
       p.status {
         @include inline-block();
@@ -250,6 +240,13 @@ section.problem {
       }
     }
 
+    .reload
+    {
+      float:right;
+      margin: 10px;
+    }
+
+
     .grader-status {
       padding: 9px;
       background: #F6F6F6;
@@ -266,6 +263,13 @@ section.problem {
         margin: -7px 7px 0 0;
       }
 
+      .grading {
+        background: url('../images/info-icon.png') left center no-repeat;
+        padding-left: 25px;
+        text-indent: 0px;
+        margin: 0px 7px 0 0;
+      }
+
       p {
         line-height: 20px;
         text-transform: capitalize;
@@ -685,6 +689,21 @@ section.problem {
             color: #B00;
           }
         }
+
+        .markup-text{
+          margin: 5px;
+          padding: 20px 0px 15px 50px;
+          border-top: 1px solid #DDD;
+          border-left: 20px solid #FAFAFA;
+
+          bs {
+            color: #BB0000;
+          }
+
+          bg {
+            color: #BDA046;
+          }
+        }
       }
     }
   }

common/lib/xmodule/xmodule/error_module.py

@@ -149,14 +149,14 @@ class ErrorDescriptor(JSONEditingDescriptor):
         '''
         try:
             xml = etree.fromstring(self.definition['data']['contents'])
-            return etree.tostring(xml)
+            return etree.tostring(xml, encoding='unicode')
         except etree.XMLSyntaxError:
             # still not valid.
             root = etree.Element('error')
             root.text = self.definition['data']['contents']
             err_node = etree.SubElement(root, 'error_msg')
             err_node.text = self.definition['data']['error_msg']
-            return etree.tostring(root)
+            return etree.tostring(root, encoding='unicode')
 
 
 class NonStaffErrorDescriptor(ErrorDescriptor):

common/lib/xmodule/xmodule/html_module.py

@@ -6,15 +6,14 @@ import sys
 from lxml import etree
 from path import path
 
-from .x_module import XModule
 from pkg_resources import resource_string
-from .xml_module import XmlDescriptor, name_to_pathname
-from .editing_module import EditingDescriptor
-from .stringify import stringify_children
-from .html_checker import check_html
-from xmodule.modulestore import Location
-
 from xmodule.contentstore.content import XASSET_SRCREF_PREFIX, StaticContent
+from xmodule.editing_module import EditingDescriptor
+from xmodule.html_checker import check_html
+from xmodule.modulestore import Location
+from xmodule.stringify import stringify_children
+from xmodule.x_module import XModule
+from xmodule.xml_module import XmlDescriptor, name_to_pathname
 
 log = logging.getLogger("mitx.courseware")
 
@@ -121,7 +120,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
 
             try:
                 with system.resources_fs.open(filepath) as file:
-                    html = file.read()
+                    html = file.read().decode('utf-8')
                     # Log a warning if we can't parse the file, but don't error
                     if not check_html(html):
                         msg = "Couldn't parse html in {0}.".format(filepath)
@@ -162,7 +161,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
 
         resource_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True)
         with resource_fs.open(filepath, 'w') as file:
-            file.write(self.definition['data'])
+            file.write(self.definition['data'].encode('utf-8'))
 
         # write out the relative name
         relname = path(pathname).basename()

common/lib/xmodule/xmodule/js/src/capa/schematic.js

@@ -1953,7 +1953,7 @@ cktsim = (function() {
 	var module = {
 	    'Circuit': Circuit,
 	    'parse_number': parse_number,
-	    'parse_source': parse_source,
+	    'parse_source': parse_source
 	}
 	return module;
     }());
@@ -2068,7 +2068,7 @@ schematic = (function() {
 	    'n': [NFet, 'NFet'],
 	    'p': [PFet, 'PFet'],
 	    's': [Probe, 'Voltage Probe'],
-	    'a': [Ammeter, 'Current Probe'],
+	    'a': [Ammeter, 'Current Probe']
 	};
 
 	// global clipboard
@@ -5502,7 +5502,7 @@ schematic = (function() {
 	    'magenta' : 'rgb(255,64,255)',
 	    'yellow': 'rgb(255,255,64)',
 	    'black': 'rgb(0,0,0)',
-	    'x-axis': undefined,
+	    'x-axis': undefined
 	};
 
 	function Probe(x,y,rotation,color,offset) {
@@ -6100,7 +6100,7 @@ schematic = (function() {
 		    'Amplitude',
 		    'Frequency (Hz)',
 		    'Delay until sin starts (secs)',
-		    'Phase offset (degrees)'],
+		    'Phase offset (degrees)']
 	}
 
 	// build property editor div
@@ -6300,7 +6300,7 @@ schematic = (function() {
 
 	var module = {
 	    'Schematic': Schematic,
-	    'component_slider': component_slider,
+	    'component_slider': component_slider
 	}
 	return module;
     }());

common/lib/xmodule/xmodule/js/src/selfassessment/display.coffee

@@ -120,7 +120,7 @@ class @SelfAssessment
     if @state == 'done'
       $.postWithPrefix "#{@ajax_url}/reset", {}, (response) =>
         if response.success
-          @answer_area.html('')
+          @answer_area.val('')
           @rubric_wrapper.html('')
           @hint_wrapper.html('')
           @message_wrapper.html('')

common/lib/xmodule/xmodule/js/src/video/display.coffee

@@ -2,6 +2,8 @@ class @Video
   constructor: (element) ->
     @el = $(element).find('.video')
     @id = @el.attr('id').replace(/video_/, '')
+    @start = @el.data('start')
+    @end = @el.data('end')
     @caption_data_dir = @el.data('caption-data-dir')
     @caption_asset_path = @el.data('caption-asset-path')
     @show_captions = @el.data('show-captions') == "true"

common/lib/xmodule/xmodule/js/src/video/display/video_player.coffee

@@ -36,14 +36,21 @@ class @VideoPlayer extends Subview
       @volumeControl = new VideoVolumeControl el: @$('.secondary-controls')
     @speedControl = new VideoSpeedControl el: @$('.secondary-controls'), speeds: @video.speeds, currentSpeed: @currentSpeed()
     @progressSlider = new VideoProgressSlider el: @$('.slider')
+    @playerVars =
+      controls: 0
+      wmode: 'transparent'
+      rel: 0
+      showinfo: 0
+      enablejsapi: 1
+      modestbranding: 1
+    if @video.start
+      @playerVars.start = @video.start
+    if @video.end
+      # work in AS3, not HMLT5. but iframe use AS3
+      @playerVars.end = @video.end
+
     @player = new YT.Player @video.id,
-      playerVars:
-        controls: 0
-        wmode: 'transparent'
-        rel: 0
-        showinfo: 0
-        enablejsapi: 1
-        modestbranding: 1
+      playerVars: @playerVars
       videoId: @video.youtubeId()
       events:
         onReady: @onReady

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -352,6 +352,12 @@ class ModuleStore(object):
         course_filter = Location("i4x", category="course")
         return self.get_items(course_filter)
 
+    def get_course(self, course_id):
+        '''
+        Look for a specific course id.  Returns the course descriptor, or None if not found.
+        '''
+        raise NotImplementedError
+
     def get_parent_locations(self, location):
         '''Find all locations that are the parents of this location.  Needed
         for path_to_location().
@@ -413,3 +419,10 @@ class ModuleStoreBase(ModuleStore):
 
         errorlog = self._get_errorlog(location)
         return errorlog.errors
+
+    def get_course(self, course_id):
+        """Default impl--linear search through course list"""
+        for c in self.get_courses():
+            if c.id == course_id:
+                return c
+        return None

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -157,7 +157,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                 make_name_unique(xml_data)
 
                 descriptor = XModuleDescriptor.load_from_xml(
-                    etree.tostring(xml_data), self, self.org,
+                    etree.tostring(xml_data, encoding='unicode'), self, self.org,
                     self.course, xmlstore.default_class)
             except Exception as err:
                 print err, self.load_error_modules
@@ -419,7 +419,7 @@ class XMLModuleStore(ModuleStoreBase):
                 self.load_error_modules,
             )
 
-            course_descriptor = system.process_xml(etree.tostring(course_data))
+            course_descriptor = system.process_xml(etree.tostring(course_data, encoding='unicode'))
 
             # NOTE: The descriptors end up loading somewhat bottom up, which
             # breaks metadata inheritance via get_children().  Instead
@@ -513,6 +513,18 @@ class XMLModuleStore(ModuleStoreBase):
         raise NotImplementedError("XMLModuleStores can't guarantee that definitions"
                                   " are unique. Use get_instance.")
 
+    def get_items(self, location, depth=0):
+        items = []
+        for _, modules in self.modules.iteritems():
+            for mod_loc, module in modules.iteritems():
+
+                # Locations match if each value in `location` is None or if the value from `location`
+                # matches the value from `mod_loc`
+                if all(goal is None or goal == value for goal, value in zip(location, mod_loc)):
+                    items.append(module)
+
+        return items
+
 
     def get_courses(self, depth=0):
         """

common/lib/xmodule/xmodule/raw_module.py

@@ -13,7 +13,7 @@ class RawDescriptor(XmlDescriptor, XMLEditingDescriptor):
     """
     @classmethod
     def definition_from_xml(cls, xml_object, system):
-        return {'data': etree.tostring(xml_object, pretty_print=True)}
+        return {'data': etree.tostring(xml_object, pretty_print=True,encoding='unicode')}
 
     def definition_to_xml(self, resource_fs):
         try:

common/lib/xmodule/xmodule/self_assessment_module.py

@@ -7,20 +7,21 @@ Parses xml definition file--see below for exact format.
 
 import copy
 from fs.errors import ResourceNotFoundError
+import itertools
+import json
 import logging
-import os
-import sys
 from lxml import etree
 from lxml.html import rewrite_links
 from path import path
-import json
-from progress import Progress
+import os
+import sys
 
 from pkg_resources import resource_string
 
 from .capa_module import only_one, ComplexEncoder
 from .editing_module import EditingDescriptor
 from .html_checker import check_html
+from progress import Progress
 from .stringify import stringify_children
 from .x_module import XModule
 from .xml_module import XmlDescriptor
@@ -52,6 +53,8 @@ class SelfAssessmentModule(XModule):
          submissions too.)
     """
 
+    STATE_VERSION = 1
+
     # states
     INITIAL = 'initial'
     ASSESSING = 'assessing'
@@ -102,35 +105,130 @@ class SelfAssessmentModule(XModule):
         else:
             instance_state = {}
 
-        # Note: score responses are on scale from 0 to max_score
-        self.student_answers = instance_state.get('student_answers', [])
-        self.scores = instance_state.get('scores', [])
-        self.hints = instance_state.get('hints', [])
+        instance_state = self.convert_state_to_current_format(instance_state)
+
+        # History is a list of tuples of (answer, score, hint), where hint may be
+        # None for any element, and score and hint can be None for the last (current)
+        # element.
+        # Scores are on scale from 0 to max_score
+        self.history = instance_state.get('history', [])
 
         self.state = instance_state.get('state', 'initial')
 
+        self.attempts = instance_state.get('attempts', 0)
+        self.max_attempts = int(self.metadata.get('attempts', MAX_ATTEMPTS))
+
         # Used for progress / grading.  Currently get credit just for
         # completion (doesn't matter if you self-assessed correct/incorrect).
-
         self._max_score = int(self.metadata.get('max_score', MAX_SCORE))
 
-        self.attempts = instance_state.get('attempts', 0)
-
-        self.max_attempts = int(self.metadata.get('attempts', MAX_ATTEMPTS))
-
         self.rubric = definition['rubric']
         self.prompt = definition['prompt']
         self.submit_message = definition['submitmessage']
         self.hint_prompt = definition['hintprompt']
 
+
+    def latest_answer(self):
+        """None if not available"""
+        if not self.history:
+            return None
+        return self.history[-1].get('answer')
+
+    def latest_score(self):
+        """None if not available"""
+        if not self.history:
+            return None
+        return self.history[-1].get('score')
+
+    def latest_hint(self):
+        """None if not available"""
+        if not self.history:
+            return None
+        return self.history[-1].get('hint')
+
+    def new_history_entry(self, answer):
+        self.history.append({'answer': answer})
+
+    def record_latest_score(self, score):
+        """Assumes that state is right, so we're adding a score to the latest
+        history element"""
+        self.history[-1]['score'] = score
+
+    def record_latest_hint(self, hint):
+        """Assumes that state is right, so we're adding a score to the latest
+        history element"""
+        self.history[-1]['hint'] = hint
+
+
+    def change_state(self, new_state):
+        """
+        A centralized place for state changes--allows for hooks.  If the
+        current state matches the old state, don't run any hooks.
+        """
+        if self.state == new_state:
+            return
+
+        self.state = new_state
+
+        if self.state == self.DONE:
+            self.attempts += 1
+
+    @staticmethod
+    def convert_state_to_current_format(old_state):
+        """
+        This module used to use a problematic state representation. This method
+        converts that into the new format.
+
+        Args:
+            old_state: dict of state, as passed in.  May be old.
+
+        Returns:
+            new_state: dict of new state
+        """
+        if old_state.get('version', 0) == SelfAssessmentModule.STATE_VERSION:
+            # already current
+            return old_state
+
+        # for now, there's only one older format.
+
+        new_state = {'version': SelfAssessmentModule.STATE_VERSION}
+
+        def copy_if_present(key):
+            if key in old_state:
+                new_state[key] = old_state[key]
+
+        for to_copy in ['attempts', 'state']:
+            copy_if_present(to_copy)
+
+        # The answers, scores, and hints need to be kept together to avoid them
+        # getting out of sync.
+
+        # NOTE: Since there's only one problem with a few hundred submissions
+        # in production so far, not trying to be smart about matching up hints
+        # and submissions in cases where they got out of sync.
+
+        student_answers = old_state.get('student_answers', [])
+        scores = old_state.get('scores', [])
+        hints = old_state.get('hints', [])
+
+        new_state['history'] = [
+            {'answer': answer,
+             'score': score,
+             'hint': hint}
+             for answer, score, hint in itertools.izip_longest(
+                     student_answers, scores, hints)]
+        return new_state
+
+
     def _allow_reset(self):
         """Can the module be reset?"""
         return self.state == self.DONE and self.attempts < self.max_attempts
 
     def get_html(self):
         #set context variables and render template
-        if self.state != self.INITIAL and self.student_answers:
-            previous_answer = self.student_answers[-1]
+        if self.state != self.INITIAL:
+            latest = self.latest_answer()
+            previous_answer = latest if latest is not None else ''
         else:
             previous_answer = ''
 
@@ -149,26 +247,19 @@ class SelfAssessmentModule(XModule):
         # cdodge: perform link substitutions for any references to course static content (e.g. images)
         return rewrite_links(html, self.rewrite_content_links)
 
-    def get_score(self):
-        """
-        Returns dict with 'score' key
-        """
-        return {'score': self.get_last_score()}
-
     def max_score(self):
         """
         Return max_score
         """
         return self._max_score
 
-    def get_last_score(self):
+    def get_score(self):
         """
         Returns the last score in the list
         """
-        last_score=0
-        if(len(self.scores)>0):
-            last_score=self.scores[len(self.scores)-1]
-        return last_score
+        score = self.latest_score()
+        return {'score': score if score is not None else 0,
+                'total': self._max_score}
 
     def get_progress(self):
         '''
@@ -176,7 +267,7 @@ class SelfAssessmentModule(XModule):
         '''
         if self._max_score > 0:
             try:
-                return Progress(self.get_last_score(), self._max_score)
+                return Progress(self.get_score()['score'], self._max_score)
             except Exception as err:
                 log.exception("Got bad progress")
                 return None
@@ -250,9 +341,10 @@ class SelfAssessmentModule(XModule):
         if self.state in (self.INITIAL, self.ASSESSING):
             return ''
 
-        if self.state == self.DONE and len(self.hints) > 0:
+        if self.state == self.DONE:
             # display the previous hint
-            hint = self.hints[-1]
+            latest = self.latest_hint()
+            hint = latest if latest is not None else ''
         else:
             hint = ''
 
@@ -281,6 +373,14 @@ class SelfAssessmentModule(XModule):
     def save_answer(self, get):
         """
         After the answer is submitted, show the rubric.
+
+        Args:
+            get: the GET dictionary passed to the ajax request.  Should contain
+                a key 'student_answer'
+
+        Returns:
+            Dictionary with keys 'success' and either 'error' (if not success),
+            or 'rubric_html' (if success).
         """
         # Check to see if attempts are less than max
         if self.attempts > self.max_attempts:
@@ -295,8 +395,9 @@ class SelfAssessmentModule(XModule):
         if self.state != self.INITIAL:
             return self.out_of_sync_error(get)
 
-        self.student_answers.append(get['student_answer'])
-        self.state = self.ASSESSING
+        # add new history element with answer and empty score and hint.
+        self.new_history_entry(get['student_answer'])
+        self.change_state(self.ASSESSING)
 
         return {
             'success': True,
@@ -318,27 +419,24 @@ class SelfAssessmentModule(XModule):
         'message_html' only if success is true
         """
 
-        n_answers = len(self.student_answers)
-        n_scores = len(self.scores)
-        if (self.state != self.ASSESSING or n_answers !=  n_scores + 1):
-            msg = "%d answers, %d scores" % (n_answers, n_scores)
-            return self.out_of_sync_error(get, msg)
+        if self.state != self.ASSESSING:
+            return self.out_of_sync_error(get)
 
         try:
             score = int(get['assessment'])
-        except:
+        except ValueError:
             return {'success': False, 'error': "Non-integer score value"}
 
-        self.scores.append(score)
+        self.record_latest_score(score)
 
         d = {'success': True,}
 
         if score == self.max_score():
-            self.state = self.DONE
+            self.change_state(self.DONE)
             d['message_html'] = self.get_message_html()
             d['allow_reset'] = self._allow_reset()
         else:
-            self.state = self.REQUEST_HINT
+            self.change_state(self.REQUEST_HINT)
             d['hint_html'] = self.get_hint_html()
 
         d['state'] = self.state
@@ -360,19 +458,15 @@ class SelfAssessmentModule(XModule):
             # the same number of hints and answers.
             return self.out_of_sync_error(get)
 
-        self.hints.append(get['hint'].lower())
-        self.state = self.DONE
-
-        # increment attempts
-        self.attempts = self.attempts + 1
+        self.record_latest_hint(get['hint'])
+        self.change_state(self.DONE)
 
         # To the tracking logs!
         event_info = {
             'selfassessment_id': self.location.url(),
             'state': {
-                'student_answers': self.student_answers,
-                'score': self.scores,
-                'hints': self.hints,
+                'version': self.STATE_VERSION,
+                'history': self.history,
                 }
             }
         self.system.track_function('save_hint', event_info)
@@ -397,7 +491,7 @@ class SelfAssessmentModule(XModule):
                 'success': False,
                 'error': 'Too many attempts.'
             }
-        self.state = self.INITIAL
+        self.change_state(self.INITIAL)
         return {'success': True}
 
 
@@ -407,12 +501,11 @@ class SelfAssessmentModule(XModule):
         """
 
         state = {
-                 'student_answers': self.student_answers,
-                 'hints': self.hints,
+                 'version': self.STATE_VERSION,
+                 'history': self.history,
                  'state': self.state,
-                 'scores': self.scores,
                  'max_score': self._max_score,
-                 'attempts': self.attempts
+                 'attempts': self.attempts,
         }
         return json.dumps(state)
 

common/lib/xmodule/xmodule/seq_module.py

@@ -10,7 +10,7 @@ from xmodule.progress import Progress
 from xmodule.exceptions import NotFoundError
 from pkg_resources import resource_string
 
-log = logging.getLogger("mitx.common.lib.seq_module")
+log = logging.getLogger(__name__)
 
 # HACK: This shouldn't be hard-coded to two types
 # OBSOLETE: This obsoletes 'type'
@@ -126,8 +126,8 @@ class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor):
         children = []
         for child in xml_object:
             try:
-                children.append(system.process_xml(etree.tostring(child)).location.url())
-            except Exception, e:
+                children.append(system.process_xml(etree.tostring(child, encoding='unicode')).location.url())
+            except Exception as e:
                 log.exception("Unable to load child when parsing Sequence. Continuing...")
                 if system.error_tracker is not None:
                     system.error_tracker("ERROR: " + str(e))

common/lib/xmodule/xmodule/stringify.py

@@ -22,7 +22,7 @@ def stringify_children(node):
     #                 next element.
     parts = [node.text]
     for c in node.getchildren():
-        parts.append(etree.tostring(c, with_tail=True))
+        parts.append(etree.tostring(c, with_tail=True, encoding='unicode'))
 
     # filter removes possible Nones in texts and tails
-    return ''.join(filter(None, parts))
+    return u''.join(filter(None, parts))

common/lib/xmodule/xmodule/tests/__init__.py

@@ -4,7 +4,7 @@ unittests for xmodule
 Run like this:
 
     rake test_common/lib/xmodule
-    
+
 """
 
 import unittest
@@ -19,11 +19,12 @@ import xmodule
 from xmodule.x_module import ModuleSystem
 from mock import Mock
 
-i4xs = ModuleSystem(
+test_system = ModuleSystem(
     ajax_url='courses/course_id/modx/a_location',
     track_function=Mock(),
     get_module=Mock(),
-    render_template=Mock(),
+    # "render" to just the context...
+    render_template=lambda template, context: str(context),
     replace_urls=Mock(),
     user=Mock(),
     filestore=Mock(),

common/lib/xmodule/xmodule/tests/test_progress.py

@@ -5,7 +5,7 @@ import unittest
 from xmodule.progress import Progress
 from xmodule import x_module
 
-from . import i4xs
+from . import test_system
 
 class ProgressTest(unittest.TestCase):
     ''' Test that basic Progress objects work.  A Progress represents a
@@ -133,6 +133,6 @@ class ModuleProgressTest(unittest.TestCase):
     '''
     def test_xmodule_default(self):
         '''Make sure default get_progress exists, returns None'''
-        xm = x_module.XModule(i4xs, 'a://b/c/d/e', None, {})
+        xm = x_module.XModule(test_system, 'a://b/c/d/e', None, {})
         p = xm.get_progress()
         self.assertEqual(p, None)

common/lib/xmodule/xmodule/tests/test_self_assessment.py

@@ -0,0 +1,54 @@
+import json
+from mock import Mock
+import unittest
+
+from xmodule.self_assessment_module import SelfAssessmentModule
+from xmodule.modulestore import Location
+
+from . import test_system
+
+class SelfAssessmentTest(unittest.TestCase):
+
+    definition = {'rubric': 'A rubric',
+                  'prompt': 'Who?',
+                  'submitmessage': 'Shall we submit now?',
+                  'hintprompt': 'Consider this...',
+                  }
+
+    location = Location(["i4x", "edX", "sa_test", "selfassessment",
+                         "SampleQuestion"])
+
+    metadata = {'attempts': '10'}
+
+    descriptor = Mock()
+
+    def test_import(self):
+        state = json.dumps({'student_answers': ["Answer 1", "answer 2", "answer 3"],
+                            'scores': [0, 1],
+                            'hints': ['o hai'],
+                            'state': SelfAssessmentModule.ASSESSING,
+                            'attempts': 2})
+
+        module = SelfAssessmentModule(test_system, self.location,
+                                      self.definition, self.descriptor,
+                                      state, {}, metadata=self.metadata)
+
+        self.assertEqual(module.get_score()['score'], 0)
+
+        self.assertTrue('answer 3' in module.get_html())
+        self.assertFalse('answer 2' in module.get_html())
+
+        module.save_assessment({'assessment': '0'})
+        self.assertEqual(module.state, module.REQUEST_HINT)
+
+        module.save_hint({'hint': 'hint for ans 3'})
+        self.assertEqual(module.state, module.DONE)
+
+        d = module.reset({})
+        self.assertTrue(d['success'])
+        self.assertEqual(module.state, module.INITIAL)
+
+        # if we now assess as right, skip the REQUEST_HINT state
+        module.save_answer({'student_answer': 'answer 4'})
+        module.save_assessment({'assessment': '1'})
+        self.assertEqual(module.state, module.DONE)

common/lib/xmodule/xmodule/video_module.py

@@ -10,6 +10,9 @@ from xmodule.modulestore.mongo import MongoModuleStore
 from xmodule.modulestore.django import modulestore
 from xmodule.contentstore.content import StaticContent
 
+import datetime
+import time
+
 log = logging.getLogger(__name__)
 
 
@@ -37,6 +40,7 @@ class VideoModule(XModule):
         self.show_captions = xmltree.get('show_captions', 'true')
         self.source = self._get_source(xmltree)
         self.track = self._get_track(xmltree)
+        self.start_time, self.end_time = self._get_timeframe(xmltree)
 
         if instance_state is not None:
             state = json.loads(instance_state)
@@ -46,11 +50,11 @@ class VideoModule(XModule):
     def _get_source(self, xmltree):
         # find the first valid source
         return self._get_first_external(xmltree, 'source')
-        
+
     def _get_track(self, xmltree):
         # find the first valid track
         return self._get_first_external(xmltree, 'track')
-        
+
     def _get_first_external(self, xmltree, tag):
         """
         Will return the first valid element
@@ -65,6 +69,23 @@ class VideoModule(XModule):
                 break
         return result
 
+    def _get_timeframe(self, xmltree):
+        """ Converts 'from' and 'to' parameters in video tag to seconds.
+        If there are no parameters, returns empty string. """
+
+        def parse_time(s):
+            """Converts s in '12:34:45' format to seconds. If s is
+            None, returns empty string"""
+            if s is None:
+                return ''
+            else:
+                x = time.strptime(s, '%H:%M:%S')
+                return datetime.timedelta(hours=x.tm_hour,
+                                      minutes=x.tm_min,
+                                      seconds=x.tm_sec).total_seconds()
+
+        return parse_time(xmltree.get('from')), parse_time(xmltree.get('to'))
+
     def handle_ajax(self, dispatch, get):
         '''
         Handle ajax calls to this video.
@@ -109,12 +130,14 @@ class VideoModule(XModule):
             'id': self.location.html_id(),
             'position': self.position,
             'source': self.source,
-            'track' : self.track,
+            'track': self.track,
             'display_name': self.display_name,
             # TODO (cpennington): This won't work when we move to data that isn't on the filesystem
             'data_dir': self.metadata['data_dir'],
             'caption_asset_path': caption_asset_path,
-            'show_captions': self.show_captions
+            'show_captions': self.show_captions,
+            'start': self.start_time,
+            'end': self.end_time
         })
 
 

common/lib/xmodule/xmodule/x_module.py

@@ -248,17 +248,17 @@ class XModule(HTMLSnippet):
             self._loaded_children = [c for c in children if c is not None]
 
         return self._loaded_children
-    
+
     def get_children_locations(self):
         '''
         Returns the locations of each of child modules.
-        
+
         Overriding this changes the behavior of get_children and
         anything that uses get_children, such as get_display_items.
-        
+
         This method will not instantiate the modules of the children
         unless absolutely necessary, so it is cheaper to call than get_children
-        
+
         These children will be the same children returned by the
         descriptor unless descriptor.has_dynamic_children() is true.
         '''
@@ -303,8 +303,20 @@ class XModule(HTMLSnippet):
         return '{}'
 
     def get_score(self):
-        ''' Score the student received on the problem.
-        '''
+        """
+        Score the student received on the problem, or None if there is no
+        score.
+
+        Returns:
+          dictionary
+             {'score': integer, from 0 to get_max_score(),
+              'total': get_max_score()}
+
+          NOTE (vshnayder): not sure if this was the intended return value, but
+          that's what it's doing now.  I suspect that we really want it to just
+          return a number.  Would need to change (at least) capa and
+          modx_dispatch to match if we did that.
+        """
         return None
 
     def max_score(self):
@@ -334,6 +346,19 @@ class XModule(HTMLSnippet):
             get is a dictionary-like object '''
         return ""
 
+    # cdodge: added to support dynamic substitutions of
+    # links for courseware assets (e.g. images). <link> is passed through from lxml.html parser
+    def rewrite_content_links(self, link):
+        # see if we start with our format, e.g. 'xasset:<filename>'
+        if link.startswith(XASSET_SRCREF_PREFIX):
+            # yes, then parse out the name
+            name = link[len(XASSET_SRCREF_PREFIX):]
+            loc = Location(self.location)
+            # resolve the reference to our internal 'filepath' which
+            link = StaticContent.compute_location_filename(loc.org, loc.course, name)
+
+        return link
+
 
 def policy_key(location):
     """
@@ -412,7 +437,7 @@ class XModuleDescriptor(Plugin, HTMLSnippet, ResourceTemplates):
     # cdodge: this is a list of metadata names which are 'system' metadata
     # and should not be edited by an end-user
     system_metadata_fields = ['data_dir', 'published_date', 'published_by', 'is_draft']
-    
+
     # A list of descriptor attributes that must be equal for the descriptors to
     # be equal
     equality_attributes = ('definition', 'metadata', 'location',
@@ -575,18 +600,18 @@ class XModuleDescriptor(Plugin, HTMLSnippet, ResourceTemplates):
             self,
             metadata=self.metadata
         )
-    
-    
+
+
     def has_dynamic_children(self):
         """
         Returns True if this descriptor has dynamic children for a given
         student when the module is created.
-        
+
         Returns False if the children of this descriptor are the same
-        children that the module will return for any student. 
+        children that the module will return for any student.
         """
         return False
-        
+
 
     # ================================= JSON PARSING ===========================
     @staticmethod
@@ -810,7 +835,8 @@ class ModuleSystem(object):
                  debug=False,
                  xqueue=None,
                  node_path="",
-                 anonymous_student_id=''):
+                 anonymous_student_id='',
+                 course_id=None):
         '''
         Create a closure around the system environment.
 
@@ -845,6 +871,8 @@ class ModuleSystem(object):
                          ajax results.
 
         anonymous_student_id - Used for tracking modules with student id
+
+        course_id - the course_id containing this module
         '''
         self.ajax_url = ajax_url
         self.xqueue = xqueue
@@ -857,6 +885,7 @@ class ModuleSystem(object):
         self.replace_urls = replace_urls
         self.node_path = node_path
         self.anonymous_student_id = anonymous_student_id
+        self.course_id = course_id
         self.user_is_staff = user is not None and user.is_staff
 
     def get(self, attr):

common/lib/xmodule/xmodule/xml_module.py

@@ -1,19 +1,20 @@
-from xmodule.x_module import (XModuleDescriptor, policy_key)
-from xmodule.modulestore import Location
-from lxml import etree
 import json
 import copy
 import logging
-import traceback
-from collections import namedtuple
-from fs.errors import ResourceNotFoundError
 import os
 import sys
+from collections import namedtuple
+from lxml import etree
+
+from xmodule.x_module import (XModuleDescriptor, policy_key)
+from xmodule.modulestore import Location
 
 log = logging.getLogger(__name__)
 
+# assume all XML files are persisted as utf-8.
 edx_xml_parser = etree.XMLParser(dtd_validation=False, load_dtd=False,
-                                 remove_comments=True, remove_blank_text=True)
+                                 remove_comments=True, remove_blank_text=True,
+                                 encoding='utf-8')
 
 def name_to_pathname(name):
     """
@@ -380,7 +381,7 @@ class XmlDescriptor(XModuleDescriptor):
             filepath = self.__class__._format_filepath(self.category, url_path)
             resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
             with resource_fs.open(filepath, 'w') as file:
-                file.write(etree.tostring(xml_object, pretty_print=True))
+                file.write(etree.tostring(xml_object, pretty_print=True, encoding='utf-8'))
 
             # And return just a pointer with the category and filename.
             record_object = etree.Element(self.category)
@@ -395,7 +396,7 @@ class XmlDescriptor(XModuleDescriptor):
             record_object.set('org', self.location.org)
             record_object.set('course', self.location.course)
 
-        return etree.tostring(record_object, pretty_print=True)
+        return etree.tostring(record_object, pretty_print=True, encoding='utf-8')
 
     def definition_to_xml(self, resource_fs):
         """

common/static/coffee/src/discussion/utils.coffee

@@ -249,7 +249,10 @@ class @DiscussionUtil
           $3
       else if RE_DISPLAYMATH.test(text)
         text = text.replace RE_DISPLAYMATH, ($0, $1, $2, $3) ->
-          processedText += $1 + processor("$$" + $2 + "$$", 'display')
+          #processedText += $1 + processor("$$" + $2 + "$$", 'display')
+          #bug fix, ordering is off
+          processedText =  processor("$$" + $2 + "$$", 'display') + processedText
+          processedText = $1 + processedText
           $3
       else
         processedText += text

common/static/sass/bourbon/css3/_box-sizing.scss

@@ -2,5 +2,5 @@
 //  content-box | border-box | inherit
   -webkit-box-sizing: $box;
      -moz-box-sizing: $box;
-          box-sizing: $box;
+          box-sizing: $box; *behavior: url(/static/scripts/boxsizing.htc)
 }

common/static/scripts/boxsizing.htc

@@ -0,0 +1,504 @@
+/**
+* box-sizing Polyfill
+* 
+* A polyfill for box-sizing: border-box for IE6 & IE7.
+* 
+* JScript
+* 
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Lesser General Public License as published 
+* by the Free Software Foundation, either version 3 of the License, or
+* (at your option) any later version.
+* 
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU Lesser General Public License for more details.
+* 
+* See <http://www.gnu.org/licenses/lgpl-3.0.txt>
+* 
+* @category  JScript 
+* @package   box-sizing-polyfill
+* @author    Christian Schepp Schaefer <schaepp@gmx.de> <http://twitter.com/derSchepp>
+* @copyright 2012 Christian Schepp Schaefer
+* @license   http://www.gnu.org/copyleft/lesser.html The GNU LESSER GENERAL PUBLIC LICENSE, Version 3.0
+* @link      http://github.com/Schepp/box-sizing-polyfill 
+*
+* PREFACE:
+*
+* This box-sizing polyfill is based on previous work done by Erik Arvidsson, 
+* which he published in 2002 on http://webfx.eae.net/dhtml/boxsizing/boxsizing.html.
+*
+* USAGE:
+*   
+* Add the behavior/HTC after every `box-sizing: border-box;` that you assign:
+* 
+* box-sizing: border-box;
+* *behavior: url(/scripts/boxsizing.htc);`
+* 
+* Prefix the `behavior` property with a star, like seen above, so it will only be seen by 
+* IE6 & IE7, not by IE8+ who already implement box-sizing.
+*
+* The URL to the HTC file must be relative to your HTML(!) document, not relative to your CSS.
+* That's why I'd advise you to use absolute paths like in the example.
+*
+*/
+<component lightWeight="true">
+<attach event="onpropertychange" onevent="checkPropertyChange()" />
+<attach event="ondetach" onevent="restore()" />
+<attach event="onresize" for="window" onevent="update()" />
+<script type="text/javascript">
+//<![CDATA[
+
+var viewportwidth = (typeof window.innerWidth != 'undefined' ? window.innerWidth : element.document.documentElement.clientWidth);
+
+// Shortcut for the document object
+var doc = element.document;
+
+// Buffer for multiple resize events
+var resizetimeout = null;
+
+// Don't apply box-sizing to certain elements
+var apply = false;
+switch(element.nodeName){
+  case '#comment':
+  case 'HTML':
+  case 'HEAD':
+  case 'TITLE':
+  case 'SCRIPT':
+  case 'STYLE':
+  case 'LINK':
+  case 'META':
+  break;
+
+  default:
+    apply = true;
+  break;
+}
+
+/*
+* update gets called during resize events, then waits until there are no further resize events, and finally triggers a recalculation
+*/
+function update(){
+  if(resizetimeout !== null){
+    window.clearTimeout(resizetimeout);
+  }
+  resizetimeout = window.setTimeout(function(){
+    restore();
+    try {
+        init();
+    }
+    catch (err) {}
+    resizetimeout = null;
+  },100);
+}
+
+/*
+* restore gets called when the behavior is being detached (see event binding at the top),
+* resets everything like it was before applying the behavior
+*/
+function restore(){
+  if(apply){
+    try{
+      element.runtimeStyle.removeAttribute("width");
+      element.runtimeStyle.removeAttribute("height");
+    }
+    catch(e){}
+  }
+}
+
+/*
+* init gets called once at the start and then never again, 
+* triggers box-sizing calculations and updates width and height
+*/
+function init(){
+  if(apply){
+    updateBorderBoxWidth();
+    updateBorderBoxHeight();
+  }
+}
+
+/*
+* checkPropertyChange gets called as soon as an element property changes 
+* (see event binding at the top), it then checks if any property influencing its 
+* dimensions was changed and if yes recalculates width and height 
+*/
+function checkPropertyChange(){
+  if(apply){
+    var pn = event.propertyName;
+    if(pn === "style.boxSizing" && element.style.boxSizing === ""){
+      element.style.removeAttribute("boxSizing");
+      element.runtimeStyle.removeAttribute("boxSizing");
+      element.runtimeStyle.removeAttribute("width");
+      element.runtimeStyle.removeAttribute("height");
+    }
+    switch (pn){
+      case "style.width":
+      case "style.minWidth":
+      case "style.maxWidth":
+      case "style.borderLeftWidth":
+      case "style.borderLeftStyle":
+      case "style.borderRightWidth":
+      case "style.borderRightStyle":
+      case "style.paddingLeft":
+      case "style.paddingRight":
+        updateBorderBoxWidth();
+      break;
+
+      case "style.height":
+      case "style.minHeight":
+      case "style.maxHeight":
+      case "style.borderTopWidth":
+      case "style.borderTopStyle":
+      case "style.borderBottomWidth":
+      case "style.borderBottomStyle":
+      case "style.paddingTop":
+      case "style.paddingBottom":
+        updateBorderBoxHeight();
+      break;
+
+      case "className":
+      case "style.boxSizing":
+        updateBorderBoxWidth();
+        updateBorderBoxHeight();
+      break;
+    }
+  }
+}
+
+/* 
+ * Helper function, taken from Dean Edward's IE7 framework,
+ * added by Schepp on 12.06.2010.
+ * http://code.google.com/p/ie7-js/
+ *
+ * Allows us to convert from relative to pixel-values.
+ */
+function getPixelValue(value){
+  var PIXEL = /^\d+(px)?$/i;
+  if (PIXEL.test(value)) return parseInt(value);
+  var style = element.style.left;
+  var runtimeStyle = element.runtimeStyle.left;
+  element.runtimeStyle.left = element.currentStyle.left;
+  element.style.left = value || 0;
+  value = parseInt(element.style.pixelLeft);
+  element.style.left = style;
+  element.runtimeStyle.left = runtimeStyle;
+
+  return value;
+}
+
+function getPixelWidth(object, value){
+  // For Pixel Values
+  var PIXEL = /^\d+(px)?$/i;
+  if (PIXEL.test(value)) return parseInt(value);
+
+  // For Percentage Values
+  var PERCENT = /^[\d\.]+%$/i;
+  if (PERCENT.test(value)){
+    try{
+      var parentPaddingLeft = getPixelWidth(object.parentElement,object.parentElement.currentStyle.paddingLeft);
+      var parentPaddingRight = getPixelWidth(object.parentElement,object.parentElement.currentStyle.paddingRight);
+      var parentBorderLeft = getPixelWidth(object.parentElement,object.parentElement.currentStyle.borderLeft);
+      var parentBorderRight = getPixelWidth(object.parentElement,object.parentElement.currentStyle.borderRight);
+
+      //var parentWidth = getPixelWidth(object.parentElement,(object.parentElement.currentStyle.width != "auto" ? object.parentElement.currentStyle.width : "100%"));
+      var parentWidth = object.parentElement.offsetWidth - parentPaddingLeft - parentPaddingRight - parentBorderLeft - parentBorderRight;
+      var value = (parseFloat(value) / 100) * parentWidth;
+    }
+    catch(e){
+      var value = (parseFloat(value) / 100) * element.document.documentElement.clientWidth;
+    }
+    return parseInt(value);
+  }
+
+  // For EM Values
+  var style = object.style.left;
+  var runtimeStyle = object.runtimeStyle.left;
+  object.runtimeStyle.left = object.currentStyle.left;
+  object.style.left = value || 0;
+  value = parseInt(object.style.pixelLeft);
+  object.style.left = style;
+  object.runtimeStyle.left = runtimeStyle;
+
+  return value;
+}
+
+function getPixelHeight(object, value){
+  // For Pixel Values
+  var PIXEL = /^\d+(px)?$/i;
+  if (PIXEL.test(value)) return parseInt(value);
+
+  // For Percentage Values
+  var PERCENT = /^[\d\.]+%$/i;
+  if (PERCENT.test(value)){
+    try{
+      if(object.parentElement.currentStyle.height != "auto"){
+        switch(object.parentElement.nodeName){
+          default:
+            if(object.parentElement.currentStyle.height !== "auto"){
+              var parentPaddingTop = getPixelWidth(object.parentElement,object.parentElement.currentStyle.paddingTop);
+              var parentPaddingBottom = getPixelWidth(object.parentElement,object.parentElement.currentStyle.paddingBottom);
+              var parentBorderTop = getPixelWidth(object.parentElement,object.parentElement.currentStyle.borderTop);
+              var parentBorderBottom = getPixelWidth(object.parentElement,object.parentElement.currentStyle.borderBottom);
+
+              var parentHeight = object.parentElement.offsetHeight - parentPaddingTop - parentPaddingBottom - parentBorderTop - parentBorderBottom;
+              //var parentHeight = getPixelHeight(object.parentElement,object.parentElement.currentStyle.height);
+
+              value = (parseFloat(value) / 100) * parentHeight;
+            }
+            else {
+              value = "auto";
+            }
+          break;
+
+          case 'HTML':
+            parentHeight = element.document.documentElement.clientHeight;
+            if(parentHeight !== "auto"){
+              value = (parseFloat(value) / 100) * parentHeight;
+            }
+            else {
+              value = "auto";
+            }
+          break;
+        }
+        if(value !== "auto") value = parseInt(value);
+      }
+      else {
+        value = "auto";
+      }
+    }
+    catch(e){
+      value = "auto";
+    }
+    return value;
+  }
+
+  // For EM Values
+  var style = object.style.left;
+  var runtimeStyle = object.runtimeStyle.left;
+  object.runtimeStyle.left = object.currentStyle.left;
+  object.style.left = value || 0;
+  value = parseInt(object.style.pixelLeft);
+  object.style.left = style;
+  object.runtimeStyle.left = runtimeStyle;
+
+  return value;
+}
+
+
+/*
+ * getBorderWidth & friends
+ * Border width getters
+ */
+function getBorderWidth(sSide){
+  if(element.currentStyle["border" + sSide + "Style"] == "none"){
+    return 0;
+  }
+  var n = getPixelValue(element.currentStyle["border" + sSide + "Width"]);
+  return n || 0;
+}
+function getBorderLeftWidth() { return getBorderWidth("Left"); }
+function getBorderRightWidth() { return getBorderWidth("Right"); }
+function getBorderTopWidth() { return getBorderWidth("Top"); }
+function getBorderBottomWidth() { return getBorderWidth("Bottom"); }
+
+
+/*
+ * getPadding & friends
+ * Padding width getters
+ */
+function getPadding(sSide) {
+  var n = getPixelValue(element.currentStyle["padding" + sSide]);
+  return n || 0;
+}
+function getPaddingLeft() { return getPadding("Left"); }
+function getPaddingRight() { return getPadding("Right"); }
+function getPaddingTop() { return getPadding("Top"); }
+function getPaddingBottom() { return getPadding("Bottom"); }
+
+
+
+/*
+ * getBoxSizing
+ * Get the box-sizing value for the current element
+ */
+function getBoxSizing(){
+  var s = element.style;
+  var cs = element.currentStyle
+  if(typeof s.boxSizing != "undefined" && s.boxSizing != ""){
+    return s.boxSizing;
+  }
+  if(typeof s["box-sizing"] != "undefined" && s["box-sizing"] != ""){
+    return s["box-sizing"];
+  }
+  if(typeof cs.boxSizing != "undefined" && cs.boxSizing != ""){
+    return cs.boxSizing;
+  }
+  if(typeof cs["box-sizing"] != "undefined" && cs["box-sizing"] != ""){
+    return cs["box-sizing"];
+  }
+  return getDocumentBoxSizing();
+}
+
+
+/*
+ * getDocumentBoxSizing
+ * Get the default document box sizing (check for quirks mode)
+ */
+function getDocumentBoxSizing(){
+  if(doc.compatMode === null || doc.compatMode === "BackCompat"){
+    return "border-box";
+  }
+  return "content-box"
+}
+
+
+/*
+ * setBorderBoxWidth & friends
+ * Width and height setters
+ */
+function setBorderBoxWidth(n){
+  element.runtimeStyle.width = Math.max(0, n - getBorderLeftWidth() -
+    getPaddingLeft() - getPaddingRight() - getBorderRightWidth()) + "px";
+}
+function setBorderBoxMinWidth(n){
+  element.runtimeStyle.minWidth = Math.max(0, n - getBorderLeftWidth() -
+    getPaddingLeft() - getPaddingRight() - getBorderRightWidth()) + "px";
+}
+function setBorderBoxMaxWidth(n){
+  element.runtimeStyle.maxWidth = Math.max(0, n - getBorderLeftWidth() -
+    getPaddingLeft() - getPaddingRight() - getBorderRightWidth()) + "px";
+}
+function setBorderBoxHeight(n){
+  element.runtimeStyle.height = Math.max(0, n - getBorderTopWidth() -
+    getPaddingTop() - getPaddingBottom() - getBorderBottomWidth()) + "px";
+}
+function setBorderBoxMinHeight(n){
+  element.runtimeStyle.minHeight = Math.max(0, n - getBorderTopWidth() -
+    getPaddingTop() - getPaddingBottom() - getBorderBottomWidth()) + "px";
+}
+function setBorderBoxMaxHeight(n){
+  element.runtimeStyle.maxHeight = Math.max(0, n - getBorderTopWidth() -
+    getPaddingTop() - getPaddingBottom() - getBorderBottomWidth()) + "px";
+}
+function setContentBoxWidth(n){
+  element.runtimeStyle.width = Math.max(0, n + getBorderLeftWidth() +
+    getPaddingLeft() + getPaddingRight() + getBorderRightWidth()) + "px";
+}
+function setContentBoxMinWidth(n){
+  element.runtimeStyle.minWidth = Math.max(0, n + getBorderLeftWidth() +
+    getPaddingLeft() + getPaddingRight() + getBorderRightWidth()) + "px";
+}
+function setContentBoxMaxWidth(n){
+  element.runtimeStyle.maxWidth = Math.max(0, n + getBorderLeftWidth() +
+    getPaddingLeft() + getPaddingRight() + getBorderRightWidth()) + "px";
+}
+function setContentBoxHeight(n){
+  element.runtimeStyle.height = Math.max(0, n + getBorderTopWidth() +
+    getPaddingTop() + getPaddingBottom() + getBorderBottomWidth()) + "px";
+}
+function setContentBoxMinHeight(n){
+  element.runtimeStyle.minHeight = Math.max(0, n + getBorderTopWidth() +
+    getPaddingTop() + getPaddingBottom() + getBorderBottomWidth()) + "px";
+}
+function setContentBoxMaxHeight(n){
+  element.runtimeStyle.maxHeight = Math.max(0, n + getBorderTopWidth() +
+    getPaddingTop() + getPaddingBottom() + getBorderBottomWidth()) + "px";
+}
+
+
+/*
+ * updateBorderBoxWidth & updateBorderBoxHeight
+ * 
+ */
+function updateBorderBoxWidth() {
+  if(getDocumentBoxSizing() == getBoxSizing()){
+    return;
+  }
+
+  var csw = element.currentStyle.width;
+  if(csw != "auto"){
+    csw = getPixelWidth(element,csw);
+    if(getBoxSizing() == "border-box"){
+      setBorderBoxWidth(parseInt(csw));
+    }
+    else{
+      setContentBoxWidth(parseInt(csw));
+    }
+  }
+
+  csw = element.currentStyle.minWidth;
+  if(csw != "none"){
+    csw = getPixelWidth(element,csw);
+    if(getBoxSizing() == "border-box"){
+      setBorderBoxMinWidth(parseInt(csw));
+    }
+    else{
+      setContentBoxMinWidth(parseInt(csw));
+    }
+  }
+
+  csw = element.currentStyle.maxWidth;
+  if(csw != "none"){
+    csw = getPixelWidth(element,csw);
+    if(getBoxSizing() == "border-box"){
+      setBorderBoxMaxWidth(parseInt(csw));
+    }
+    else{
+      setContentBoxMaxWidth(parseInt(csw));
+    }
+  }
+}
+
+function updateBorderBoxHeight() {
+  if(getDocumentBoxSizing() == getBoxSizing()){
+    return;
+  }
+
+  var csh = element.currentStyle.height;
+  if(csh != "auto"){
+    csh = getPixelHeight(element,csh);
+    if(csh !== "auto"){
+      if(getBoxSizing() == "border-box"){
+        setBorderBoxHeight(parseInt(csh));
+      }
+      else{
+        setContentBoxHeight(parseInt(csh));
+      }
+    }
+  }
+
+  csh = element.currentStyle.minHeight;
+  if(csh != "none"){
+    csh = getPixelHeight(element,csh);
+    if(csh !== "none"){
+      if(getBoxSizing() == "border-box"){
+        setBorderBoxMinHeight(parseInt(csh));
+      }
+      else{
+        setContentBoxMinHeight(parseInt(csh));
+      }
+    }
+  }
+
+  csh = element.currentStyle.maxHeight;
+  if(csh != "none"){
+    csh = getPixelHeight(element,csh);
+    if(csh !== "none"){
+      if(getBoxSizing() == "border-box"){
+        setBorderBoxMaxHeight(parseInt(csh));
+      }
+      else{
+        setContentBoxMaxHeight(parseInt(csh));
+      }
+    }
+  }
+}
+
+
+// Run the calculations
+init();
+
+//]]>
+</script>
+</component>

common/templates/mathjax_include.html

@@ -33,4 +33,4 @@
 <!-- This must appear after all mathjax-config blocks, so it is after the imports from the other templates.
      It can't be run through static.url because MathJax uses crazy url introspection to do lazy loading of
      MathJax extension libraries -->
-<script type="text/javascript" src="/static/js/vendor/mathjax-MathJax-c9db6ac/MathJax.js?config=TeX-MML-AM_HTMLorMML-full"></script>
+<script type="text/javascript" src="https://edx-static.s3.amazonaws.com/mathjax-MathJax-07669ac/MathJax.js?config=TeX-MML-AM_HTMLorMML-full"></script>

common/test/data/full/about/faq.html

@@ -6,7 +6,7 @@
     <p>No - anyone and everyone is welcome to take this course.</p>
   </li>
   <li>What textbook should I buy?
-    <p>Although the lectures are designed to be self-contained, we recommend (but do not require) that students refer to the book Worlds Together, Worlds Apart: A History of the World: From 1000 CE to the Present (W W Norton, 3rd edition) -- Volume II, which was written specifically for this course.</p>
+    <p>Although the lectures are designed to be self-contained, we recommend (but do not require) that students refer to the book Worlds Together, Worlds Apart: A History of the World: From 1000 CE to the Present (W W Norton, 3rd edition) &mdash; Volume II, which was written specifically for this course.</p>
   </li>
   <li>Does Harvard award credentials or reports regarding my work in this course?
     <p>Princeton does not award credentials or issue reports for student work in this course. However, Coursera could maintain a record of your score on the assessments and, with your permission, verify that score for authorized parties.</p>

common/test/data/full/chapter/Overview.xml

@@ -2,7 +2,7 @@
   <video url_name="welcome"/>
   <sequential filename="System_Usage_Sequence" slug="System_Usage_Sequence" format="Lecture Sequence" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="System Usage Sequence"/>
   <vertical slug="Lab0_Using_the_tools" format="Lab" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="Lab0: Using the tools">
-    <html slug="html_19" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab </html>
+    <html slug="html_19" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab… </html>
     <html slug="html_5555" filename="html_5555"/>
     <problem filename="Lab_0_Using_the_Tools" slug="Lab_0_Using_the_Tools" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="false" name="Lab 0: Using the Tools"/>
   </vertical>

common/test/data/full/custom_tags/book

@@ -1 +1 @@
-More information given in <a href="/book/${page}">the text</a>. 
+More information given in… <a href="/book/${page}">the text</a>. 

common/test/data/full/custom_tags/discuss

@@ -1 +1 @@
-<a href='https://6002x.mitx.mit.edu/discussion/questions/scope:all/sort:activity-desc/tags:${tag}/page:1/'> Discussion: ${tag} </a>
+<a href='https://6002x.mitx.mit.edu/discussion/questions/scope:all/sort:activity-desc/tags:${tag}/page:1/'> Discussion: ${tag}… </a>

common/test/data/full/custom_tags/slides

@@ -1 +1 @@
-Lecture Slides Handout [<a href="">Clean </a>][<a href="">Annotated</a>]
+Lecture Slides Handout [<a href="">Clean… </a>][<a href="">Annotated…</a>]

common/test/data/full/html/FirstOrderSystemHint.xml

@@ -1,4 +1,4 @@
-Hint
+Hint…
 <br/><br/>
 Remember that the time evolution of any variable \(x(t)\) governed by
 a first-order system with a time-constant \(\tau\) for a time \(t) between an initial

common/test/data/full/html/Week_13_Tutorials.html

@@ -4,14 +4,14 @@
     <section class="tutorials">
       <h2> Basic Tutorials </h2>
       <ul>
-        <li><a href="/section/wk13_solder">Soldering</a> -- Steve
+        <li><a href="/section/wk13_solder">Soldering</a> &mdash; Steve
        Finberg, one of the pioneers in from Draper Lab, talks about
        soldering. </li>
       </ul>
       <h2> Bonus Tutorials </h2>
       <ul>
         <li><a href="/section/wk13_FreqResp">Frequency Response
-       Curves</a> -- We explain several techniques for understanding
+       Curves</a> &mdash; We explain several techniques for understanding
        and approximating Bode plots. </li>
       </ul>
     </section>

common/test/data/full/html/Week_1_Tutorials.html

@@ -41,7 +41,7 @@
         <li><a href="/section/problem_1_3">OCW Problem 1-3 </a> - Reverse engineer a black-box resistor network</li>
       </ul>
       <hr/>
-      <p>  Since the course has students from a diverse set of backgrounds, the first week's tutorials includes several extra segments, worked out with greater detail, to help bring everyone up to speed. </p>
+      <p>  Since the course has students from a diverse set of backgrounds, the first week's tutorials includes several extra segments, worked out with greater detail, to help bring everyone up to speed. Gratuitous &ge; entity.</p>
     </section>
   </body>
 </html>

common/test/data/full/html/html_5555.html

@@ -1 +1 @@
-    <html slug="html_5555" filename="html_5555> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab </html>
+    <html slug="html_5555" filename="html_5555> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab. </html>

common/test/data/full/html/linearity_clarify.html

@@ -34,6 +34,6 @@
     the Thevenin or Norton theorems to summarize the behavior at
     a pair of exposed terminals.
     </p><p>
-    Sorry for the confusion of words -- natural language is like
+    Sorry for the confusion of words &mdash; natural language is like
     that!
   </p>

common/test/data/full/html/linearity_clarify.xml

@@ -34,6 +34,6 @@
     the Thevenin or Norton theorems to summarize the behavior at
     a pair of exposed terminals.
     </p><p>
-    Sorry for the confusion of words -- natural language is like
+    Sorry for the confusion of words &mdash; natural language is like
     that!
   </p>

common/test/data/full/html/schematic_tutorial.html

@@ -9,14 +9,14 @@ the right of the diagram area) and drag it onto the diagram.  Release
 the mouse when the component is in the correct position.
 </td>
 </tr>
-
+<!-- note that entities like &mdash; may be used. -->
 <tr>
 <td>Move a component</td>
 <td>Click to select a component in the diagram (it will turn green)
 and then drag it to its new location.  You can use shift-click to add
 a component to the current selection.  Or you can click somewhere in
 the diagram that is not on top of a component and drag out a selection
-rectangle -- components intersecting the rectangle will be added to
+rectangle &mdash; components intersecting the rectangle will be added to
 the current selection.
 </td>
 </tr>
@@ -63,7 +63,7 @@ engineeering notation:
 <td>Add a wire</td>
 <td>Wires start at connection points, the open circles that
 appear at the terminals of components or the ends of wires.
-Click on a connection point to start a wire -- a green wire
+Click on a connection point to start a wire &mdash; a green wire
 will appear with one end anchored at the starting point.
 Drag the mouse and release the mouse button when the other
 end of the wire is positioned as you wish.  Once a wire has

common/test/data/full/html/units_hint.html

@@ -1,4 +1,4 @@
-Hint
+Hint…
 <br/><br/>
 Be careful of units here.  Make sure you notice multipliers such
-as u, k, m, M.
+as u (or &mu;), k, m, M.

common/test/data/full/info/updates.html

@@ -9,8 +9,9 @@
 <li> <h2>May 2 </h2>
 <section class="update-description">
 <ul>
-<li> We have opened the show-answer button on the midterm. </li>
-<li> There was a four hour outage in posting ability on the discussion board Monday night. It has been fixed. We apologise for the inconvenience.</li>
+<!-- utf-8 characters are acceptable… as are HTML entities -->
+<li> We have opened the show-answer button on the midterm… </li>
+<li> There was a four hour outage in posting ability on the discussion board Monday night&hellip; It has been fixed. We apologise for the inconvenience.</li>
 </ul>
 </li>
 <li> <h2>April 30 </h2>

common/test/data/full/problem/Circuit_Sandbox.xml

@@ -1,6 +1,6 @@
-<problem><startouttext/><p/>Here's a sandbox where you can experiment with all the components
+<problem><!-- include ellipses to test non-ascii characters --><startouttext/><p/>Here's a sandbox where you can experiment with all the components
 we'll discuss in 6.002x.  If you click on CHECK below, your diagram
-will be saved on the server and you can return at some later time.
+will be saved on the server and you can return at some later time…
 <endouttext/><schematicresponse><p/><center><schematic name="work" value="" width="800" height="600"/></center><answer type="loncapa/python">
 correct = ['correct']
 </answer></schematicresponse></problem>

common/test/data/full/problem/H1P3_Poor_Workmanship.xml

@@ -78,7 +78,8 @@ So the total heating power in Joe's shop was:
 <numericalresponse answer="$Pbad"><responseparam type="tolerance" default="5%" name="tol" description="Numerical Tolerance"/><textline/></numericalresponse>
 <startouttext/>
 <br/>
-No wonder Joe was cold.
+<!-- add non-ascii utf-8 character here -->
+No wonder Joe was cold…
 <endouttext/>
 
 </problem>

common/test/data/full/problem/Lab_0_Using_the_Tools.xml

@@ -94,7 +94,7 @@ scope probes to nodes A, B and C and edit their properties so that the
 plots will be different colors. Now run a transient analysis for 5ms.
 Move the mouse over the plot until the marker (a vertical dashed line
 that follows the mouse when it's over the plot) is at approximately
-1.25ms.  Please report the measured voltages for nodes A, B and C.
+1.25ms.  Please report the measured voltages for nodes A, B and C…
 
 <br/>
 <div style="margin-left: 4em;">

common/test/data/full/problem/Sample_Algebraic_Problem.xml

@@ -6,7 +6,7 @@ z = "A*x^2 + sqrt(y)"
 Enter the algebraic expression \(A x^2 + \sqrt{y}\) in the box below.  The
 entry is case sensitive.  The product must be indicated with an
 asterisk, and the exponentation with a caret, so you must write 
-"A*x^2 + sqrt(y)".
+"A*x^2 + sqrt(y)"…
 <endouttext/>
 <formularesponse type="cs" samples="A,x,y@1,1,1:3,3,3#10" answer="$z"><responseparam description="Numerical Tolerance" type="tolerance" default="0.00001" name="tol"/><textline size="40"/></formularesponse>
 

common/test/data/full/problem/Sample_Numeric_Problem.xml

@@ -1,6 +1,6 @@
 <problem><startouttext/>
 Enter the numerical value of the expression \(x + y\) where 
-\(x = 3\) and \(y = 5\).
+\(x = 3\) and \(y = 5\)…
 <endouttext/>
 
 <numericalresponse answer="8"><responseparam type="tolerance" default="5%" name="tol" description="Numerical Tolerance"/><textline/></numericalresponse>

common/test/data/full/problem/choiceresponse_demo.xml

@@ -1,19 +1,20 @@
 <problem display_name="S3E2: Lorentz Force">
 
 <startouttext/>
-<p>Consider a hypothetical magnetic field pointing out of your computer screen. Now imagine an electron traveling from right to leftin the plane of your screen. A diagram of this situation is show below.</p>
+<p>Consider a hypothetical magnetic field pointing out of your computer screen. Now imagine an electron traveling from right to left in the plane of your screen. A diagram of this situation is show below…</p>
 <center><img width="400" src="/static/images/LSQimages/LSQ_W01_8.png"/></center>
 
 <p>a. The magnitude of the force experienced by the electron is proportional the product of which of the following? (Select all that apply.)</p>
 <endouttext/>
 <choiceresponse>
 <checkboxgroup>
-<choice correct="true"><text>Magnetic field strength</text></choice>
-<choice correct="false"><text>Electric field strength</text></choice>
-<choice correct="true"><text>Electric charge of the electron</text></choice>
-<choice correct="false"><text>Radius of the electron</text></choice>
-<choice correct="false"><text>Mass of the electron</text></choice>
-<choice correct="true"><text>Velocity of the electron</text></choice>
+<!-- include ellipses to test non-ascii characters -->
+<choice correct="true"><text>Magnetic field strength…</text></choice>
+<choice correct="false"><text>Electric field strength…</text></choice>
+<choice correct="true"><text>Electric charge of the electron…</text></choice>
+<choice correct="false"><text>Radius of the electron…</text></choice>
+<choice correct="false"><text>Mass of the electron…</text></choice>
+<choice correct="true"><text>Velocity of the electron…</text></choice>
 
 </checkboxgroup>
 </choiceresponse>

common/test/data/full/problem/codeinput_demo.xml

@@ -2,7 +2,8 @@
 <problem display_name="L4 Problem 1">
   <text>
     <p>
-      <b class="bfseries">Part 1: Function Types</b>
+      <!-- include ellipses to test non-ascii characters -->
+      <b class="bfseries">Part 1: Function Types…</b>
     </p>
     <p>
 For each of the following functions, specify the type of its <b class="bfseries">output</b>. You can assume each function is called with an appropriate argument, as specified by its docstring. </p>

common/test/data/full/sequential/Administrivia_and_Circuit_Elements.xml

@@ -3,12 +3,13 @@
   <vertical slug="vertical_66" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never">
     <problem filename="S1E3_AC_power" slug="S1E3_AC_power" name="S1E3: AC power"/>
     <customtag tag="S1E3" slug="discuss_67" impl="discuss"/>
-    <html slug="html_68"> S1E4 has been removed. </html>
+    <!-- utf-8 characters acceptable, but not HTML entities -->
+    <html slug="html_68"> S1E4 has been removed…</html>
   </vertical>
   <vertical filename="vertical_89" slug="vertical_89" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"/>
   <vertical slug="vertical_94" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never">
     <video youtube="0.75:XNh13VZhThQ,1.0:XbDRmF6J0K0,1.25:JDty12WEQWk,1.50:wELKGj-5iyM" slug="What_s_next" name="What's next"/>
-    <html slug="html_95">Minor correction: Six elements (five resistors)</html>
+    <html slug="html_95">Minor correction: Six elements (five resistors)…</html>
     <customtag tag="S1" slug="discuss_96" impl="discuss"/>
   </vertical>
 </sequential>

common/test/data/full/vertical/vertical_89.xml

@@ -1,6 +1,7 @@
 <sequential>
   <html slug="html_90">
-    <h1> </h1>
+    <!-- UTF-8 characters are acceptable… HTML entities are not -->
+    <h1>Inline content…</h1>
   </html>
   <video youtube="1.50:vl9xrfxcr38,1.25:qxNX4REGqx4,1.0:BGU1poJDgOY,0.75:8rK9vnpystQ" slug="S1V14_Summary" name="S1V14: Summary"/>
   <customtag tag="S1" slug="discuss_91" impl="discuss"/>

common/test/data/full/video/welcome.xml

@@ -1 +1 @@
-<video youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8" format="Video" display_name="Welcome"/>
+<video youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8" format="Video" display_name="Welcome…"/>

common/test/data/toy/html/secret/toylab.html

@@ -1,3 +1,13 @@
 <b>Lab 2A: Superposition Experiment</b>
 
+<<<<<<< Updated upstream
 <p>Isn't the toy course great?</p>
+
+<p>Let's add some markup that uses non-ascii characters.
+For example, we should be able to write words like encyclop&aelig;dia, or foreign words like fran&ccedil;ais.
+Looking beyond latin-1, we should handle math symbols:  &pi;r&sup2 &le; &#8734.
+And it shouldn't matter if we use entities or numeric codes &mdash; &Omega; &ne; &pi; &equiv; &#937; &#8800; &#960;.
+</p>
+=======
+<p>Isn't the toy course great? — &le;</p>
+>>>>>>> Stashed changes

create-dev-env.sh

@@ -90,16 +90,17 @@ clone_repos() {
     fi
 }
 
+### START
+
 PROG=${0##*/}
 BASE="$HOME/mitx_all"
 PYTHON_DIR="$BASE/python"
 RUBY_DIR="$BASE/ruby"
 RUBY_VER="1.9.3"
-NUMPY_VER="1.6.2"
-SCIPY_VER="0.10.1"
-BREW_FILE="$BASE/mitx/brew-formulas.txt"
 LOG="/var/tmp/install-$(date +%Y%m%d-%H%M%S).log"
-APT_PKGS="pkg-config curl git python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor nodejs npm graphviz graphviz-dev mysql-server libmysqlclient-dev"
+
+
+# Read arguments
 
 if [[ $EUID -eq 0 ]]; then
     error "This script should not be run using sudo or as the root user"
@@ -162,18 +163,14 @@ info
 output "Press return to begin or control-C to abort"
 read dummy
 
-# log all stdout and stderr
+
+# Log all stdout and stderr
+
 exec > >(tee $LOG)
 exec 2>&1
 
-if ! grep -q "export rvm_path=$RUBY_DIR" ~/.rvmrc; then
-    if [[ -f $HOME/.rvmrc ]]; then
-        output "Copying existing .rvmrc to .rvmrc.bak"
-        cp $HOME/.rvmrc $HOME/.rvmrc.bak
-    fi
-    output "Creating $HOME/.rvmrc so rvm uses $RUBY_DIR"
-    echo "export rvm_path=$RUBY_DIR" > $HOME/.rvmrc
-fi
+
+# Install basic system requirements
 
 mkdir -p $BASE
 case `uname -s` in
@@ -182,17 +179,11 @@ case `uname -s` in
             error "Please install lsb-release."
             exit 1
         }
+
         distro=`lsb_release -cs`
         case $distro in
             maya|lisa|natty|oneiric|precise|quantal)
-                output "Installing ubuntu requirements"
-                sudo apt-get install python-software-properties
-                sudo add-apt-repository ppa:chris-lea/node.js
-                sudo apt-get -y update
-                # DEBIAN_FRONTEND=noninteractive is required for silent mysql-server installation
-                sudo DEBIAN_FRONTEND=noninteractive apt-get -y install $APT_PKGS
-                sudo npm install coffee-script
-                clone_repos
+                sudo apt-get install git
                 ;;
             *)
                 error "Unsupported distribution - $distro"
@@ -200,8 +191,8 @@ case `uname -s` in
                ;;
         esac
         ;;
-    Darwin)
 
+    Darwin)
         if [[ ! -w /usr/local ]]; then
             cat<<EO
 
@@ -228,39 +219,6 @@ EO
             brew install git
         }
 
-        clone_repos
-
-        output "Installing OSX requirements"
-        if [[ ! -r $BREW_FILE ]]; then
-            error "$BREW_FILE does not exist, needed to install brew deps"
-            exit 1
-        fi
-        # brew errors if the package is already installed
-        for pkg in $(cat $BREW_FILE); do
-            grep $pkg <(brew list) &>/dev/null || {
-                output "Installing $pkg"
-                brew install $pkg
-            }
-        done
-
-        # paths where brew likes to install python scripts
-        PATH=/usr/local/share/python:/usr/local/bin:$PATH
-
-        command -v pip &>/dev/null || {
-            output "Installing pip"
-            easy_install pip
-        }
-
-        if ! grep -Eq ^1.7 <(virtualenv --version 2>/dev/null); then
-            output "Installing virtualenv >1.7"
-            pip install 'virtualenv>1.7' virtualenvwrapper
-        fi
-
-        command -v coffee &>/dev/null || {
-            output "Installing coffee script"
-            curl --insecure https://npmjs.org/install.sh | sh
-            npm install -g coffee-script
-        }
         ;;
     *)
         error "Unsupported platform"
@@ -268,19 +226,54 @@ EO
         ;;
 esac
 
+
+# Clone MITx repositories
+
+clone_repos
+
+
+# Install system-level dependencies
+
+bash $BASE/mitx/install-system-req.sh
+
+
+# Install Ruby RVM
+
 output "Installing rvm and ruby"
+
+if ! grep -q "export rvm_path=$RUBY_DIR" ~/.rvmrc; then
+    if [[ -f $HOME/.rvmrc ]]; then
+        output "Copying existing .rvmrc to .rvmrc.bak"
+        cp $HOME/.rvmrc $HOME/.rvmrc.bak
+    fi
+    output "Creating $HOME/.rvmrc so rvm uses $RUBY_DIR"
+    echo "export rvm_path=$RUBY_DIR" > $HOME/.rvmrc
+fi
+
 curl -sL get.rvm.io | bash -s -- --version 1.15.7
 source $RUBY_DIR/scripts/rvm
-# skip the intro
-LESS="-E" rvm install $RUBY_VER
+LESS="-E" rvm install $RUBY_VER --with-readline
+
 output "Installing gem bundler"
 gem install bundler
+
 output "Installing ruby packages"
 # hack :(
 cd $BASE/mitx  || true
 bundle install
 
-cd $BASE
+
+# Install Python virtualenv
+
+output "Installing python virtualenv"
+
+case `uname -s` in
+    Darwin)
+        # Add brew's path
+        PATH=/usr/local/share/python:/usr/local/bin:$PATH
+        ;;
+esac
+
 if [[ $systempkgs ]]; then
     virtualenv --system-site-packages "$PYTHON_DIR"
 else
@@ -289,9 +282,14 @@ else
     virtualenv  "$PYTHON_DIR"
 fi
 
-# change to mitx python virtualenv
+# activate mitx python virtualenv
 source $PYTHON_DIR/bin/activate
 
+# compile numpy and scipy if requested
+
+NUMPY_VER="1.6.2"
+SCIPY_VER="0.10.1"
+
 if [[ -n $compile ]]; then
     output "Downloading numpy and scipy"
     curl -sL -o numpy.tar.gz http://downloads.sourceforge.net/project/numpy/NumPy/${NUMPY_VER}/numpy-${NUMPY_VER}.tar.gz
@@ -323,18 +321,25 @@ case `uname -s` in
 esac
 
 output "Installing MITx pre-requirements"
-pip install -r mitx/pre-requirements.txt
-# Need to be in the mitx dir to get the paths to local modules right
+pip install -r $BASE/mitx/pre-requirements.txt
+
 output "Installing MITx requirements"
-cd mitx
+# Need to be in the mitx dir to get the paths to local modules right
+cd $BASE/mitx
 pip install -r requirements.txt
 
 mkdir "$BASE/log" || true
 mkdir "$BASE/db" || true
 
+
+# Configure Git
+
 output "Fixing your git default settings"
 git config --global push.default current
 
+
+### DONE
+
 cat<<END
    Success!!
 

doc/development.md

@@ -83,6 +83,7 @@ To run a single nose test:
 
 Very handy: if you uncomment the `--pdb` argument in `NOSE_ARGS` in `lms/envs/test.py`, it will drop you into pdb on error.  This lets you go up and down the stack and see what the values of the variables are.  Check out http://docs.python.org/library/pdb.html
 
+
 ### Javascript Tests
 
 These commands start a development server with jasmine testing enabled, and launch your default browser
@@ -105,6 +106,15 @@ Run the following to see a list of all rake tasks available and their arguments
 
     rake -T
 
+## Testing using queue servers
+
+When testing problems that use a queue server on AWS (e.g. sandbox-xqueue.edx.org), you'll need to run your server on your public IP, like so.
+
+`django-admin.py runserver --settings=lms.envs.dev --pythonpath=. 0.0.0.0:8000`
+
+When you connect to the LMS, you need to use the public ip.  Use `ifconfig` to figure out the numnber, and connect e.g. to `http://18.3.4.5:8000/`
+
+
 ## Content development
 
 If you change course content, while running the LMS in dev mode, it is unnecessary to restart to refresh the modulestore.  

doc/xml-format.md

@@ -418,6 +418,10 @@ If you want to customize the courseware tabs displayed for your course, specify
 * "external_link".  Parameters "name", "link".
 * "textbooks".  No parameters--generates tab names from book titles.
 * "progress".  Parameter "name".
+* "static_tab".  Parameters "name", 'url_slug'--will look for tab contents in
+                       'tabs/{course_url_name}/{tab url_slug}.html'
+* "staff_grading". No parameters.  If specified, displays the staff grading tab for instructors.
+
 
 # Tips for content developers
 
@@ -429,9 +433,7 @@ before the week 1 material to make it easy to find in the file.
 
 * Come up with a consistent pattern for url_names, so that it's easy to know where to look for any piece of content.  It will also help to come up with a standard way of splitting your content files.  As a point of departure, we suggest splitting chapters, sequences, html, and problems into separate files.
 
-* A heads up: our content management system will allow you to develop content through a web browser, but will be backed by this same xml at first.  Once that happens, every element will be in its own file to make access and updates faster.
-
-* Prefer the most "semantic" name for containers: e.g., use problemset rather than vertical for a problem set.  That way, if we decide to display problem sets differently, we don't have to change the xml.
+* Prefer the most "semantic" name for containers: e.g., use problemset rather than sequential for a problem set.  That way, if we decide to display problem sets differently, we don't have to change the xml.
 
 # Other file locations (info and about)
 

docs/source/capa.rst

@@ -1,9 +1,15 @@
 *******************************************
 Capa module
 *******************************************
+Contents:
 
 .. module:: capa
 
+.. toctree::
+    :maxdepth: 2
+
+    chem.rst
+
 Calc
 ====
 

docs/source/chem.rst

@@ -0,0 +1,69 @@
+*******************************************
+Chem module
+*******************************************
+
+.. module:: chem
+
+Miller
+======
+
+.. automodule:: capa.chem.miller
+    :members:
+    :show-inheritance:
+
+UI part and inputtypes
+----------------------
+Miller module is used in the system in crystallography problems.
+Crystallography is a class in :mod:`capa` inputtypes module.
+It uses *crystallography.html* for rendering and **crystallography.js**
+for UI part.
+
+Documentation from **crystallography.js**::
+
+    For a crystallographic problem of the type
+
+     Given a plane definition via miller indexes, specify it by plotting points on the edges
+     of a 3D cube. Additionally, select the correct Bravais cubic lattice type depending on the
+     physical crystal mentioned in the problem.
+
+    we create a graph which contains a cube, and a 3D Cartesian coordinate system. The interface
+    will allow to plot 3 points anywhere along the edges of the cube, and select which type of
+    Bravais lattice should be displayed along with the basic cube outline.
+
+    When 3 points are successfully plotted, an intersection of the resulting plane (defined by
+    the 3 plotted points), and the cube, will be automatically displayed for clarity.
+
+    After lotting the three points, it is possible to continue plotting additional points. By
+    doing so, the point that was plotted first (from the three that already exist), will be
+    removed, and the new point will be added. The intersection of the resulting new plane and
+    the cube will be redrawn.
+
+    The UI has been designed in such a way, that the user is able to determine which point will
+    be removed next (if adding a new point). This is achieved via filling the to-be-removed point
+    with a different color.
+
+
+
+Chemcalc
+========
+
+.. automodule:: capa.chem.chemcalc
+    :members:
+    :show-inheritance:
+
+Chemtools
+=========
+
+.. automodule:: capa.chem.chemtools
+    :members:
+    :show-inheritance:
+
+
+Tests
+=====
+
+.. automodule:: capa.chem.tests
+    :members:
+    :show-inheritance:
+
+

install-system-req.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+
+# posix compliant sanity check
+if [ -z $BASH ] || [  $BASH = "/bin/sh" ]; then
+    echo "Please use the bash interpreter to run this script"
+    exit 1
+fi
+
+error() {
+      printf '\E[31m'; echo "$@"; printf '\E[0m'
+}
+output() {
+      printf '\E[36m'; echo "$@"; printf '\E[0m'
+}
+
+
+### START
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+BREW_FILE=$DIR/"brew-formulas.txt"
+APT_REPOS_FILE=$DIR/"apt-repos.txt"
+APT_PKGS_FILE=$DIR/"apt-packages.txt"
+
+case `uname -s` in
+    [Ll]inux)
+        command -v lsb_release &>/dev/null || {
+            error "Please install lsb-release."
+            exit 1
+        }
+
+        distro=`lsb_release -cs`
+        case $distro in
+            maya|lisa|natty|oneiric|precise|quantal)
+                output "Installing Ubuntu requirements"
+
+                # DEBIAN_FRONTEND=noninteractive is required for silent mysql-server installation
+                export DEBIAN_FRONTEND=noninteractive
+
+                # add repositories
+                cat $APT_REPOS_FILE | xargs -n 1 sudo add-apt-repository -y
+                sudo apt-get -y update
+
+                # install packages listed in APT_PKGS_FILE
+                cat $APT_PKGS_FILE | xargs sudo apt-get -y install
+                ;;
+            *)
+                error "Unsupported distribution - $distro"
+                exit 1
+               ;;
+        esac
+        ;;
+    Darwin)
+
+        if [[ ! -w /usr/local ]]; then
+            cat<<EO
+
+        You need to be able to write to /usr/local for
+        the installation of brew and brew packages.
+
+        Either make sure the group you are in (most likely 'staff')
+        can write to that directory or simply execute the following
+        and re-run the script:
+
+        $ sudo chown -R $USER /usr/local
+EO
+
+            exit 1
+
+        fi
+
+        output "Installing OSX requirements"
+        if [[ ! -r $BREW_FILE ]]; then
+            error "$BREW_FILE does not exist, needed to install brew"
+            exit 1
+        fi
+
+        # brew errors if the package is already installed
+        for pkg in $(cat $BREW_FILE); do
+            grep $pkg <(brew list) &>/dev/null || {
+                output "Installing $pkg"
+                brew install $pkg
+            }
+        done
+
+        # paths where brew likes to install python scripts
+        PATH=/usr/local/share/python:/usr/local/bin:$PATH
+
+        command -v pip &>/dev/null || {
+            output "Installing pip"
+            easy_install pip
+        }
+
+        if ! grep -Eq ^1.7 <(virtualenv --version 2>/dev/null); then
+            output "Installing virtualenv >1.7"
+            pip install 'virtualenv>1.7' virtualenvwrapper
+        fi
+
+        command -v coffee &>/dev/null || {
+            output "Installing coffee script"
+            curl --insecure https://npmjs.org/install.sh | sh
+            npm install -g coffee-script
+        }
+        ;;
+    *)
+        error "Unsupported platform"
+        exit 1
+        ;;
+esac

jenkins/test.sh

@@ -3,6 +3,21 @@
 set -e
 set -x
 
+function github_status {
+    gcli status create mitx mitx $GIT_COMMIT \
+         --params=$1 \
+                  target_url:$BUILD_URL \
+                  description:"Build #$BUILD_NUMBER $2" \
+         -f csv
+}
+
+function github_mark_failed_on_exit {
+    trap '[ $? == "0" ] || github_status state:failure "failed"' EXIT
+}
+
+github_mark_failed_on_exit
+github_status state:pending "is running"
+
 # Reset the submodule, in case it changed
 git submodule foreach 'git reset --hard HEAD'
 
@@ -12,8 +27,8 @@ export PYTHONIOENCODING=UTF-8
 GIT_BRANCH=${GIT_BRANCH/HEAD/master}
 
 pip install -q -r pre-requirements.txt
+pip install -q -r test-requirements.txt
 yes w | pip install -q -r requirements.txt
-[ ! -d askbot ] || pip install -q -r askbot/askbot_requirements.txt
 
 rake clobber
 TESTS_FAILED=0
@@ -27,4 +42,6 @@ rake phantomjs_jasmine_common/lib/xmodule || true
 rake coverage:xml coverage:html
 
 [ $TESTS_FAILED == '0' ]
-rake autodeploy_properties
+rake autodeploy_properties
+
+github_status state:success "passed"

jenkins/test_lms.sh

@@ -1,27 +0,0 @@
-#! /bin/bash
-
-set -e
-set -x
-
-# Reset the submodule, in case it changed
-git submodule foreach 'git reset --hard HEAD'
-
-# Set the IO encoding to UTF-8 so that askbot will start
-export PYTHONIOENCODING=UTF-8
-
-GIT_BRANCH=${GIT_BRANCH/HEAD/master}
-
-pip install -q -r pre-requirements.txt
-yes w | pip install -q -r requirements.txt
-[ ! -d askbot ] || pip install -q -r askbot/askbot_requirements.txt
-
-rake clobber
-TESTS_FAILED=0
-rake test_lms[false] || TESTS_FAILED=1
-rake test_common/lib/capa || TESTS_FAILED=1
-rake test_common/lib/xmodule || TESTS_FAILED=1
-rake phantomjs_jasmine_lms || true
-rake coverage:xml coverage:html
-
-[ $TESTS_FAILED == '0' ]
-rake autodeploy_properties

lms/.coveragerc

@@ -2,11 +2,13 @@
 [run]
 data_file = reports/lms/.coverage
 source = lms
+omit = lms/envs/*
 
 [report]
 ignore_errors = True
 
 [html]
+title = LMS Python Test Coverage Report
 directory = reports/lms/cover
 
 [xml]

lms/djangoapps/certificates/management/commands/find_unicode_certs.py

@@ -1,38 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from django.core.management.base import BaseCommand
-from certificates.models import certificate_status_for_student
-from certificates.queue import XQueueCertInterface
-from django.contrib.auth.models import User
-from student.models import UserProfile
-
-
-class Command(BaseCommand):
-
-    help = """
-    Looks for names that have unicode characters
-    and queues them up for a certificate request
-    """
-
-    def handle(self, *args, **options):
-
-        # TODO this is only temporary for CS169 certs
-
-        course_id = 'BerkeleyX/CS169.1x/2012_Fall'
-
-        enrolled_students = User.objects.filter(
-                courseenrollment__course_id=course_id).prefetch_related(
-                        "groups").order_by('username')
-        xq = XQueueCertInterface()
-        print "Looking for unusual names.."
-        for student in enrolled_students:
-            if certificate_status_for_student(
-                     student, course_id)['status'] == 'unavailable':
-                continue
-            name = UserProfile.objects.get(user=student).name
-            for c in name:
-                if ord(c) >= 0x200:
-                    ret = xq.add_cert(student, course_id)
-                    if ret == 'generating':
-                        print 'generating for {0}'.format(student)
-                    break

lms/djangoapps/certificates/management/commands/fix_ungraded_certs.py

@@ -0,0 +1,45 @@
+from certificates.models import GeneratedCertificate
+from courseware import grades, courses
+from django.test.client import RequestFactory
+from django.core.management.base import BaseCommand
+from optparse import make_option
+
+
+class Command(BaseCommand):
+
+    help = """
+    Find all students that need to be graded
+    and grade them.
+    """
+
+    option_list = BaseCommand.option_list + (
+        make_option('-n', '--noop',
+            action='store_true',
+            dest='noop',
+            default=False,
+            help="Print but do not update the GeneratedCertificate table"),
+        make_option('-c', '--course',
+            metavar='COURSE_ID',
+            dest='course',
+            default=False,
+            help='Grade ungraded users for this course'),
+
+        )
+
+    def handle(self, *args, **options):
+
+        course_id = options['course']
+        print "Fetching ungraded students for {0}".format(course_id)
+        ungraded = GeneratedCertificate.objects.filter(
+                course_id__exact=course_id).filter(grade__exact='')
+        course = courses.get_course_by_id(course_id)
+        factory = RequestFactory()
+        request = factory.get('/')
+
+        for cert in ungraded:
+            # grade the student
+            grade = grades.grade(cert.user, request, course)
+            print "grading {0} - {1}".format(cert.user, grade['percent'])
+            cert.grade = grade['percent']
+            if not options['noop']:
+                cert.save()

lms/djangoapps/certificates/management/commands/gen_cert_report.py

@@ -0,0 +1,98 @@
+from django.core.management.base import BaseCommand
+from certificates.models import GeneratedCertificate
+from django.contrib.auth.models import User
+from optparse import make_option
+from django.conf import settings
+from xmodule.course_module import CourseDescriptor
+from xmodule.modulestore.django import modulestore
+from django.db.models import Count
+
+
+class Command(BaseCommand):
+
+    help = """
+
+    Generate a certificate status report for all courses that have ended.
+    This command does not do anything other than report the current
+    certificate status.
+
+    unavailable  - A student is not eligible for a certificate.
+    generating   - A request has been made to generate a certificate,
+                   but it has not been generated yet.
+    regenerating - A request has been made to regenerate a certificate,
+                   but it has not been generated yet.
+    deleting     - A request has been made to delete a certificate.
+
+    deleted      - The certificate has been deleted.
+    downloadable - The certificate is available for download.
+    notpassing   - The student was graded but is not passing
+
+    """
+
+    option_list = BaseCommand.option_list + (
+        make_option('-c', '--course',
+            metavar='COURSE_ID',
+            dest='course',
+            default=None,
+            help='Only generate for COURSE_ID'),
+        )
+
+    def _ended_courses(self):
+        for course_id in [course  # all courses in COURSE_LISTINGS
+                for sub in settings.COURSE_LISTINGS
+                    for course in settings.COURSE_LISTINGS[sub]]:
+            course_loc = CourseDescriptor.id_to_location(course_id)
+            course = modulestore().get_instance(course_id, course_loc)
+            if course.has_ended():
+                yield course_id
+
+    def handle(self, *args, **options):
+
+        # Find all courses that have ended
+
+        if options['course']:
+            ended_courses = [options['course']]
+        else:
+            ended_courses = self._ended_courses()
+
+        cert_data = {}
+
+        for course_id in ended_courses:
+
+            # find students who are enrolled
+            print "Looking up certificate states for {0}".format(course_id)
+            enrolled_students = User.objects.filter(
+                    courseenrollment__course_id=course_id).prefetch_related(
+                            "groups").order_by('username')
+            unavailable_count = enrolled_students.count() - \
+                    GeneratedCertificate.objects.filter(
+                            course_id__exact=course_id).count()
+            cert_data[course_id] = {'enrolled': enrolled_students.count()}
+            cert_data[course_id].update({'unavailable': unavailable_count})
+
+            tallies = GeneratedCertificate.objects.filter(
+                        course_id__exact=course_id).values('status').annotate(
+                            dcount=Count('status'))
+            cert_data[course_id].update(
+                    {status['status']: status['dcount']
+                                        for status in tallies})
+
+        # all states we have seen far all courses
+        status_headings = set(
+                [status for course in cert_data
+                    for status in cert_data[course]])
+
+        # print the heading for the report
+        print "{:>20}".format("course ID"),
+        print ' '.join(["{:>12}".format(heading)
+                            for heading in status_headings])
+
+        # print the report
+        for course_id in cert_data:
+            print "{0:>20}".format(course_id[0:18]),
+            for heading in status_headings:
+                if heading in cert_data[course_id]:
+                    print "{:>12}".format(cert_data[course_id][heading]),
+                else:
+                    print " " * 12,
+            print

lms/djangoapps/certificates/management/commands/ungenerated_certs.py

@@ -2,29 +2,92 @@ from django.core.management.base import BaseCommand
 from certificates.models import certificate_status_for_student
 from certificates.queue import XQueueCertInterface
 from django.contrib.auth.models import User
+from optparse import make_option
+from django.conf import settings
+from xmodule.course_module import CourseDescriptor
+from xmodule.modulestore.django import modulestore
+from certificates.models import CertificateStatuses
+import datetime
 
 
 class Command(BaseCommand):
 
     help = """
-    Find all students that have need certificates
-    and put certificate requests on the queue
+    Find all students that need certificates
+    for courses that have finished and
+    put their cert requests on the queue
 
-    This is only for BerkeleyX/CS169.1x/2012_Fall
+    Use the --noop option to test without actually
+    putting certificates on the queue to be generated.
     """
 
+    option_list = BaseCommand.option_list + (
+        make_option('-n', '--noop',
+            action='store_true',
+            dest='noop',
+            default=False,
+            help="Don't add certificate requests to the queue"),
+        make_option('-c', '--course',
+            metavar='COURSE_ID',
+            dest='course',
+            default=False,
+            help='Grade and generate certificates for a specific course'),
+
+        )
+
     def handle(self, *args, **options):
 
-        # TODO This is only temporary for CS169 certs
+        # Will only generate a certificate if the current
+        # status is in this state
 
-        course_id = 'BerkeleyX/CS169.1x/2012_Fall'
-        enrolled_students = User.objects.filter(
-                courseenrollment__course_id=course_id).prefetch_related(
-                        "groups").order_by('username')
-        xq = XQueueCertInterface()
-        for student in enrolled_students:
-            if certificate_status_for_student(
-                     student, course_id)['status'] == 'unavailable':
-                ret = xq.add_cert(student, course_id)
-                if ret == 'generating':
-                    print 'generating for {0}'.format(student)
+        VALID_STATUSES = [
+                 CertificateStatuses.unavailable
+        ]
+
+        # Print update after this many students
+
+        STATUS_INTERVAL = 500
+
+        if options['course']:
+            ended_courses = [options['course']]
+        else:
+            # Find all courses that have ended
+            ended_courses = []
+            for course_id in [course  # all courses in COURSE_LISTINGS
+                    for sub in settings.COURSE_LISTINGS
+                        for course in settings.COURSE_LISTINGS[sub]]:
+                course_loc = CourseDescriptor.id_to_location(course_id)
+                course = modulestore().get_instance(course_id, course_loc)
+                if course.has_ended():
+                    ended_courses.append(course_id)
+
+        for course_id in ended_courses:
+            print "Fetching enrolled students for {0}".format(course_id)
+            enrolled_students = User.objects.filter(
+                    courseenrollment__course_id=course_id).prefetch_related(
+                            "groups").order_by('username')
+            xq = XQueueCertInterface()
+            total = enrolled_students.count()
+            count = 0
+            start = datetime.datetime.now()
+            for student in enrolled_students:
+                count += 1
+                if count % STATUS_INTERVAL == 0:
+                    # Print a status update with an approximation of
+                    # how much time is left based on how long the last
+                    # interval took
+                    diff = datetime.datetime.now() - start
+                    timeleft = diff * (total - count) / STATUS_INTERVAL
+                    hours, remainder = divmod(timeleft.seconds, 3600)
+                    minutes, seconds = divmod(remainder, 60)
+                    print "{0}/{1} completed ~{2:02}:{3:02}m remaining".format(
+                            count, total, hours, minutes)
+                    start = datetime.datetime.now()
+
+                if certificate_status_for_student(
+                     student, course_id)['status'] in VALID_STATUSES:
+                    if not options['noop']:
+                        # Add the certificate request to the queue
+                        ret = xq.add_cert(student, course_id)
+                        if ret == 'generating':
+                            print '{0} - {1}'.format(student, ret)

lms/djangoapps/certificates/models.py

@@ -75,7 +75,9 @@ def certificate_status_for_student(student, course_id):
     This returns a dictionary with a key for status, and other information.
     The status is one of the following:
 
-    unavailable  - A student is not eligible for a certificate.
+    unavailable  - No entry for this student--if they are actually in
+                   the course, they probably have not been graded for
+                   certificate generation yet.
     generating   - A request has been made to generate a certificate,
                    but it has not been generated yet.
     regenerating - A request has been made to regenerate a certificate,
@@ -90,7 +92,7 @@ def certificate_status_for_student(student, course_id):
     "download_url".
 
     If the student has been graded, the dictionary also contains their
-    grade for the course.
+    grade for the course with the key "grade".
     '''
 
     try:

lms/djangoapps/certificates/queue.py

@@ -240,7 +240,7 @@ class XQueueCertInterface(object):
                 cert.save()
             else:
                 cert_status = status.notpassing
-
+                cert.grade = grade['percent']
                 cert.status = cert_status
                 cert.user = student
                 cert.course_id = course_id

lms/djangoapps/courseware/access.py

@@ -43,7 +43,8 @@ def has_access(user, obj, action, course_context=None):
 
     user: a Django user object. May be anonymous.
 
-    obj: The object to check access for.  For now, a module or descriptor.
+    obj: The object to check access for.  A module, descriptor, location, or
+                    certain special strings (e.g. 'global')
 
     action: A string specifying the action that the client is trying to perform.
 

lms/djangoapps/courseware/courses.py

@@ -236,11 +236,51 @@ def get_courses_by_university(user, domain=None):
     '''
     # TODO: Clean up how 'error' is done.
     # filter out any courses that errored.
-    visible_courses = branding.get_visible_courses(domain)
+    visible_courses = get_courses(user, domain)
 
     universities = defaultdict(list)
     for course in visible_courses:
-        if not has_access(user, course, 'see_exists'):
-            continue
         universities[course.org].append(course)
+
     return universities
+
+
+def get_courses(user, domain=None):
+    '''
+    Returns a list of courses available, sorted by course.number
+    '''
+    courses = branding.get_visible_courses(domain)
+    courses = [c for c in courses if has_access(user, c, 'see_exists')]
+
+    # Add metadata about the start day and if the course is new
+    for course in courses:
+        days_to_start = _get_course_days_to_start(course)
+
+        metadata = course.metadata
+        metadata['days_to_start'] = days_to_start
+        metadata['is_new'] = course.metadata.get('is_new', days_to_start > 1)
+
+    courses = sorted(courses, key=lambda course:course.number)
+    return courses
+
+
+def _get_course_days_to_start(course):
+    from datetime import datetime as dt
+    from time import mktime, gmtime
+
+    convert_to_datetime = lambda ts: dt.fromtimestamp(mktime(ts))
+
+    start_date = convert_to_datetime(course.start)
+
+    # If the course has a valid advertised date, use that instead
+    advertised_start = course.metadata.get('advertised_start', None)
+    if advertised_start:
+        try:
+            start_date = dt.strptime(advertised_start, "%Y-%m-%dT%H:%M")
+        except ValueError:
+            pass # Invalid date, keep using course.start
+
+    now = convert_to_datetime(gmtime())
+    days_to_start = (start_date - now).days
+
+    return days_to_start

lms/djangoapps/courseware/module_render.py

@@ -1,4 +1,3 @@
-import hashlib
 import json
 import logging
 import pyparsing
@@ -20,6 +19,7 @@ from mitxmako.shortcuts import render_to_string
 from models import StudentModule, StudentModuleCache
 from psychometrics.psychoanalyze import make_psychometrics_data_update_handler
 from static_replace import replace_urls
+from student.models import unique_id_for_user
 from xmodule.errortracker import exc_info_to_str
 from xmodule.exceptions import NotFoundError
 from xmodule.modulestore import Location
@@ -157,12 +157,6 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
     if not has_access(user, descriptor, 'load', course_id):
         return None
 
-    # Anonymized student identifier
-    h = hashlib.md5()
-    h.update(settings.SECRET_KEY)
-    h.update(str(user.id))
-    anonymous_student_id = h.hexdigest()
-
     # Only check the cache if this module can possibly have state
     instance_module = None
     shared_module = None
@@ -235,7 +229,8 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
                           # by the replace_static_urls code below
                           replace_urls=replace_urls,
                           node_path=settings.NODE_PATH,
-                          anonymous_student_id=anonymous_student_id,
+                          anonymous_student_id=unique_id_for_user(user),
+                          course_id=course_id,
                           )
     # pass position specified in URL to module through ModuleSystem
     system.set('position', position)