From 8e7eef24a2e1a1eda91787d1c22e3053cd7b6500 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Thu, 23 Aug 2012 13:10:08 -0400
Subject: [PATCH 001/181] add manage_course_groups command to lms_migration

---
 .../commands/manage_course_groups.py          | 71 +++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 lms/djangoapps/lms_migration/management/commands/manage_course_groups.py

diff --git a/lms/djangoapps/lms_migration/management/commands/manage_course_groups.py b/lms/djangoapps/lms_migration/management/commands/manage_course_groups.py
new file mode 100644
index 0000000000..0043b483e0
--- /dev/null
+++ b/lms/djangoapps/lms_migration/management/commands/manage_course_groups.py
@@ -0,0 +1,71 @@
+#!/usr/bin/python
+#
+# File:   manage_course_groups
+#
+# interactively list and edit membership in course staff and instructor groups
+
+import os, sys, string, re
+import datetime
+from getpass import getpass
+import json
+import readline
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.contrib.auth.models import User, Group
+
+#-----------------------------------------------------------------------------
+# get all staff groups
+
+class Command(BaseCommand):
+    help = "Manage course group membership, interactively."
+
+    def handle(self, *args, **options):
+
+        gset = Group.objects.all()
+        
+        print "Groups:"
+        for cnt,g in zip(range(len(gset)), gset):
+            print "%d. %s" % (cnt,g)
+        
+        gnum = int(raw_input('Choose group to manage (enter #): '))
+        
+        group = gset[gnum]
+        
+        #-----------------------------------------------------------------------------
+        # users in group
+        
+        uall = User.objects.all()
+        print "----"
+        print "List of All Users: %s" % [str(x.username) for x in uall]
+        print "----"
+        
+        while True:
+        
+            print "Users in the group:"
+            
+            uset = group.user_set.all()
+            for cnt, u in zip(range(len(uset)), uset):
+                print "%d. %s" % (cnt, u)
+        
+            action = raw_input('Choose user to delete (enter #) or enter usernames (comma delim) to add: ')
+        
+            m = re.match('^[0-9]+$',action)
+            if m:
+                unum = int(action)
+                u = uset[unum]
+                print "Deleting user %s" % u
+                u.groups.remove(group)
+            
+            else:
+                for uname in action.split(','):
+                    try:
+                        user = User.objects.get(username=action)
+                    except Exception as err:
+                        print "Error %s" % err
+                        continue
+                    print "adding %s to group %s" % (user, group)
+                    user.groups.add(group)
+                
+            
+            

From d94ef5454999fe738cefbf9555515f23b70ed692 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Andr=C3=A9s=20Rocha?= <rocha@edx.org>
Date: Thu, 23 Aug 2012 16:30:58 -0400
Subject: [PATCH 002/181] Added cheatsheet popup to wiki editor.

---
 lms/static/sass/course/wiki/_wiki.scss      | 23 ++++++++-
 lms/templates/wiki/base.html                | 27 +++++++----
 lms/templates/wiki/includes/cheatsheet.html | 53 +++++++++++++++++++++
 3 files changed, 93 insertions(+), 10 deletions(-)
 create mode 100644 lms/templates/wiki/includes/cheatsheet.html

diff --git a/lms/static/sass/course/wiki/_wiki.scss b/lms/static/sass/course/wiki/_wiki.scss
index bcc2c8855d..ead58bf6a6 100644
--- a/lms/static/sass/course/wiki/_wiki.scss
+++ b/lms/static/sass/course/wiki/_wiki.scss
@@ -567,11 +567,30 @@ section.wiki {
     background: #f00 !important;
   }
 
-  
-
+  #cheatsheetLink {
+   text-align:right;
+   display: float;
+  }
 
+  #cheatsheetModal {
+   width: 350px;
+   margin-left: 100px;
+   margin-top: -100px;
+   }
 
+  #cheatsheet-body {
+   background: #FFF;
+   text-align: left;
+   padding: 10px;
+  }
 
+  #cheatsheet-body pre{
+   color: #000;
+   text-align: left;
+   background: #EEE;
+   margin:10px;
+   padding: 10px;
+  }
 
   /*-----------------
 
diff --git a/lms/templates/wiki/base.html b/lms/templates/wiki/base.html
index fc4a2d18d4..a346be0e3e 100644
--- a/lms/templates/wiki/base.html
+++ b/lms/templates/wiki/base.html
@@ -27,21 +27,29 @@
       });
     }
   </script>
-  
-  
+
+
   {% addtoblock 'js' %}
     {% comment %} These scripts load at the bottom of the body {% endcomment %}
-  
+
     <script src="{% static 'js/bootstrap-alert.js' %}"></script>
     <script src="{% static 'js/bootstrap-collapse.js' %}"></script>
     <script src="{% static 'js/bootstrap-modal.js' %}"></script>
-    
+
     {% with mathjax_mode='wiki' %}
       {% include "mathjax_include.html" %}
     {% endwith %}
-    
+    <script type="text/javascript">
+	    $(document).ready(function () {
+          editor = $('#div_id_content div.controls');
+          cs = editor.prepend('<a id="cheatsheetLink" href="#">cheatsheet</a>');
+          cs.click(function() {
+              $('#cheatsheetModal').modal('show');
+          });
+	    });
+    </script>
   {% endaddtoblock %}
-  
+
 {% endblock %}
 
 
@@ -64,11 +72,14 @@
             </div>
           {% endfor %}
         {% endif %}
-        
+
         {% block wiki_contents %}{% endblock %}
-        
+
       {% endblock %}
     </div>
+
+    {% include "wiki/includes/cheatsheet.html" %}
+
   </section>
 
 {% endblock %}
diff --git a/lms/templates/wiki/includes/cheatsheet.html b/lms/templates/wiki/includes/cheatsheet.html
new file mode 100644
index 0000000000..d58920b814
--- /dev/null
+++ b/lms/templates/wiki/includes/cheatsheet.html
@@ -0,0 +1,53 @@
+<section id="cheatsheetModal" class="modal hide fade" >
+<div id="cheatsheet-body" class="modal-body">
+<h2>Wiki Syntax Help</h2>
+<p> This wiki uses <strong>Markdown</strong> for styling.
+  There are several <a href="http://daringfireball.net/projects/markdown/basics" target="_blank">useful</a>
+  <a href="http://greg.vario.us/doc/markdown.txt" target="_blank">guides</a>
+  <a href="http://www.lowendtalk.com/discussion/6/miniature-markdown-guide" target="_blank">online</a>.
+<p> To create a new wiki article, create a link to it. Clicking the link gives you the creation page.
+<pre>
+[Article Name](wiki:ArticleName)
+</pre>
+<h2> Useful examples:</h2>
+<pre>
+http://wikipedia.org
+[Wikipedia](http://wikipedia.org)
+[edX Wiki](wiki:/edx/)
+</pre>
+<pre>
+Huge Header
+===========
+</pre>
+<pre>
+Smaller Header
+--------------
+</pre>
+<pre>
+*emphasis* or _emphasis_
+</pre>
+<pre>
+**strong** or __strong__
+</pre>
+<pre>
+- Unordered List
+  - Sub Item 1
+  - Sub Item 2
+</pre>
+<pre>
+1. Ordered
+2. List
+</pre>
+<pre>
+> Quotes
+</pre>
+<h2>edX Additions:</h2>
+<pre>
+circuit-schematic:
+</pre>
+<pre>
+$LaTeX Math Expression$
+</pre>
+</div>
+</div>
+</section>

From 77f928d118a3c718dd583ec4bf63bef9fad19140 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Thu, 23 Aug 2012 17:24:59 -0400
Subject: [PATCH 003/181] Queuestate records both secret key and time of queue
 request

---
 common/lib/capa/capa/capa_problem.py      |  6 +-----
 common/lib/capa/capa/correctmap.py        | 11 ++++++-----
 common/lib/capa/capa/responsetypes.py     | 14 ++++++++------
 common/lib/xmodule/xmodule/capa_module.py |  6 ++++++
 4 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py
index 82eb330174..35c8eaf635 100644
--- a/common/lib/capa/capa/capa_problem.py
+++ b/common/lib/capa/capa/capa_problem.py
@@ -202,11 +202,7 @@ class LoncapaProblem(object):
         '''
         Returns True if any part of the problem has been submitted to an external queue
         '''
-        queued = False
-        for answer_id in self.correct_map:
-            if self.correct_map.is_queued(answer_id):
-                queued = True
-        return queued
+        return any([self.correct_map.is_queued(answer_id) for answer_id in self.correct_map])
 
     def grade_answers(self, answers):
         '''
diff --git a/common/lib/capa/capa/correctmap.py b/common/lib/capa/capa/correctmap.py
index eb6ef2d00c..c6fc98e62f 100644
--- a/common/lib/capa/capa/correctmap.py
+++ b/common/lib/capa/capa/correctmap.py
@@ -15,7 +15,8 @@ class CorrectMap(object):
     - msg         : string (may have HTML) giving extra message response (displayed below textline or textbox)
     - hint        : string (may have HTML) giving optional hint (displayed below textline or textbox, above msg)
     - hintmode    : one of (None,'on_request','always') criteria for displaying hint
-    - queuekey    : a random integer for xqueue_callback verification
+    - queuestate  : Tuple (key, time) where key is a secret string, and time is a string dump
+                    of a DateTime object in the format '%Y%m%d%H%M%S'. Is None when not queued
 
     Behaves as a dict.
     '''
@@ -31,14 +32,14 @@ class CorrectMap(object):
     def __iter__(self):
         return self.cmap.__iter__()
 
-    def set(self, answer_id=None, correctness=None, npoints=None, msg='', hint='', hintmode=None, queuekey=None):
+    def set(self, answer_id=None, correctness=None, npoints=None, msg='', hint='', hintmode=None, queuestate=None):
         if answer_id is not None:
             self.cmap[answer_id] = {'correctness': correctness,
                                     'npoints': npoints,
                                     'msg': msg,
                                     'hint': hint,
                                     'hintmode': hintmode,
-                                    'queuekey': queuekey,
+                                    'queuestate': queuestate,
                                     }
 
     def __repr__(self):
@@ -67,10 +68,10 @@ class CorrectMap(object):
         return None
 
     def is_queued(self, answer_id):
-        return answer_id in self.cmap and self.cmap[answer_id]['queuekey'] is not None
+        return answer_id in self.cmap and self.cmap[answer_id]['queuestate'] is not None
 
     def is_right_queuekey(self, answer_id, test_key):
-        return answer_id in self.cmap and self.cmap[answer_id]['queuekey'] == test_key
+        return self.is_queued(answer_id) and self.cmap[answer_id]['queuestate'][0] == test_key
 
     def get_npoints(self, answer_id):
         npoints = self.get_property(answer_id, 'npoints')
diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index b2d56b48ca..35b8688a7b 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -26,6 +26,7 @@ import xml.sax.saxutils as saxutils
 # specific library imports
 from calc import evaluator, UndefinedVariable
 from correctmap import CorrectMap
+from datetime import datetime
 from util import *
 from lxml import etree
 from lxml.html.soupparser import fromstring as fromstring_bs	 # uses Beautiful Soup!!! FIXME?
@@ -1026,7 +1027,7 @@ class CodeResponse(LoncapaResponse):
         TODO: Determines whether in synchronous or asynchronous (queued) mode
         '''
         xml = self.xml
-        self.url = xml.get('url', None) # XML can override external resource (grader/queue) URL
+        self.url = xml.get('url', None) # TODO: XML can override external resource (grader/queue) URL
         self.queue_name = xml.get('queuename', self.system.xqueue['default_queuename'])
 
         # VS[compat]:
@@ -1128,7 +1129,7 @@ class CodeResponse(LoncapaResponse):
         xheader = xqueue_interface.make_xheader(lms_callback_url=self.system.xqueue['callback_url'],
                                                 lms_key=queuekey,
                                                 queue_name=self.queue_name)
-
+        
         # Generate body
         if is_list_of_files(submission):
             self.context.update({'submission': queuekey}) # For tracking. TODO: May want to record something else here
@@ -1148,16 +1149,17 @@ class CodeResponse(LoncapaResponse):
             (error, msg) = qinterface.send_to_queue(header=xheader,
                                                     body=json.dumps(contents))
 
+        queuestate = (queuekey,'')
         cmap = CorrectMap() 
         if error:
-            cmap.set(self.answer_id, queuekey=None,
+            cmap.set(self.answer_id, queuestate=None,
                      msg='Unable to deliver your submission to grader. (Reason: %s.) Please try again later.' % msg)
         else:
             # Queueing mechanism flags:
-            #   1) Backend: Non-null CorrectMap['queuekey'] indicates that the problem has been queued
+            #   1) Backend: Non-null CorrectMap['queuestate'] indicates that the problem has been queued
             #   2) Frontend: correctness='incomplete' eventually trickles down through inputtypes.textbox 
             #       and .filesubmission to inform the browser to poll the LMS
-            cmap.set(self.answer_id, queuekey=queuekey, correctness='incomplete', msg=msg)
+            cmap.set(self.answer_id, queuestate=queuestate, correctness='incomplete', msg=msg)
 
         return cmap
 
@@ -1180,7 +1182,7 @@ class CodeResponse(LoncapaResponse):
                 points = 0
             elif points > self.maxpoints[self.answer_id]:
                 points = self.maxpoints[self.answer_id]
-            oldcmap.set(self.answer_id, npoints=points, correctness=correctness, msg=msg.replace('&nbsp;', '&#160;'), queuekey=None)  # Queuekey is consumed
+            oldcmap.set(self.answer_id, npoints=points, correctness=correctness, msg=msg.replace('&nbsp;', '&#160;'), queuestate=None)  # Queuestate is consumed
         else:
             log.debug('CodeResponse: queuekey %s does not match for answer_id=%s.' % (queuekey, self.answer_id))
 
diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py
index d2ed3912a4..6b6c0991c5 100644
--- a/common/lib/xmodule/xmodule/capa_module.py
+++ b/common/lib/xmodule/xmodule/capa_module.py
@@ -462,6 +462,12 @@ class CapaModule(XModule):
             self.system.track_function('save_problem_check_fail', event_info)
             raise NotFoundError('Problem must be reset before it can be checked again')
 
+        # Problem queued. Student should not be able to submit
+        '''
+        if self.lcp.is_queued():
+            return {'success': False, 'html': 'Already queued'} 
+        '''
+
         try:
             old_state = self.lcp.get_state()
             lcp_id = self.lcp.problem_id

From 673cc2bbb393b3cff6d3f7c2e5e8cbd3837bd73f Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Thu, 23 Aug 2012 17:43:38 -0400
Subject: [PATCH 004/181] Keep queueuing time

---
 common/lib/capa/capa/correctmap.py    | 3 +++
 common/lib/capa/capa/responsetypes.py | 5 ++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/common/lib/capa/capa/correctmap.py b/common/lib/capa/capa/correctmap.py
index c6fc98e62f..4659e2d1e2 100644
--- a/common/lib/capa/capa/correctmap.py
+++ b/common/lib/capa/capa/correctmap.py
@@ -73,6 +73,9 @@ class CorrectMap(object):
     def is_right_queuekey(self, answer_id, test_key):
         return self.is_queued(answer_id) and self.cmap[answer_id]['queuestate'][0] == test_key
 
+    def get_queuetime_str(self, answer_id):
+        return self.cmap[answer_id]['queuestate'][1] if self.is_queued(answer_id) else None 
+
     def get_npoints(self, answer_id):
         npoints = self.get_property(answer_id, 'npoints')
         if npoints is not None:
diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index 35b8688a7b..27801ad871 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -1149,7 +1149,10 @@ class CodeResponse(LoncapaResponse):
             (error, msg) = qinterface.send_to_queue(header=xheader,
                                                     body=json.dumps(contents))
 
-        queuestate = (queuekey,'')
+        # State associated with the queueing request
+        qtime = datetime.strftime(datetime.now(), '%Y%m%d%H%M%S')
+        queuestate = (queuekey, qtime)
+
         cmap = CorrectMap() 
         if error:
             cmap.set(self.answer_id, queuestate=None,

From 95a7a5b08b51455463c6400666efe5acf59c7316 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Thu, 23 Aug 2012 18:39:57 -0400
Subject: [PATCH 005/181] Student must wait XQUEUE_WAITTIME_BETWEEN_REQUESTS
 between queueing submissions

---
 common/lib/capa/capa/capa_problem.py      | 16 ++++++++++++++++
 common/lib/xmodule/xmodule/capa_module.py | 13 ++++++++-----
 lms/envs/dev.py                           |  1 +
 3 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py
index 35c8eaf635..ee24ebd031 100644
--- a/common/lib/capa/capa/capa_problem.py
+++ b/common/lib/capa/capa/capa_problem.py
@@ -29,6 +29,7 @@ from xml.sax.saxutils import unescape
 
 import calc
 from correctmap import CorrectMap
+from datetime import datetime
 import eia
 import inputtypes
 from util import contextualize_text, convert_files_to_filenames
@@ -204,6 +205,21 @@ class LoncapaProblem(object):
         '''
         return any([self.correct_map.is_queued(answer_id) for answer_id in self.correct_map])
 
+
+    def get_recentmost_queuetime(self):
+        '''
+        Returns a DateTime object that represents the timestamp of the most recent queueing request, or None if not queued
+        '''
+        if not self.is_queued():
+            return None
+
+        # Get a list of timestamps of all queueing requests, then convert it to a DateTime object
+        queuetimes = [self.correct_map.get_queuetime_str(answer_id) for answer_id in self.correct_map if self.correct_map.is_queued(answer_id)]
+        queuetimes = [datetime.strptime(qt,'%Y%m%d%H%M%S') for qt in queuetimes]
+
+        return max(queuetimes)
+
+
     def grade_answers(self, answers):
         '''
         Grade student responses.  Called by capa_module.check_problem.
diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py
index 6b6c0991c5..8f570bcf6c 100644
--- a/common/lib/xmodule/xmodule/capa_module.py
+++ b/common/lib/xmodule/xmodule/capa_module.py
@@ -7,7 +7,8 @@ import traceback
 import re
 import sys
 
-from datetime import timedelta
+from datetime import datetime, timedelta
+from django.conf import settings
 from lxml import etree
 from pkg_resources import resource_string
 
@@ -462,11 +463,13 @@ class CapaModule(XModule):
             self.system.track_function('save_problem_check_fail', event_info)
             raise NotFoundError('Problem must be reset before it can be checked again')
 
-        # Problem queued. Student should not be able to submit
-        '''
+        # Problem queued. Students must wait XQUEUE_WAITTIME_BETWEEN_REQUESTS
         if self.lcp.is_queued():
-            return {'success': False, 'html': 'Already queued'} 
-        '''
+            current_time = datetime.now()
+            prev_submit_time = self.lcp.get_recentmost_queuetime() 
+            if (current_time-prev_submit_time).total_seconds() < settings.XQUEUE_WAITTIME_BETWEEN_REQUESTS:
+                msg = 'You must wait %d seconds between queueing requests' % settings.XQUEUE_WAITTIME_BETWEEN_REQUESTS
+                return {'success': msg, 'html': ''}
 
         try:
             old_state = self.lcp.get_state()
diff --git a/lms/envs/dev.py b/lms/envs/dev.py
index a76e6de262..fe12ee3322 100644
--- a/lms/envs/dev.py
+++ b/lms/envs/dev.py
@@ -63,6 +63,7 @@ XQUEUE_INTERFACE = {
     },
     "basic_auth": ('anant', 'agarwal'),
 }
+XQUEUE_WAITTIME_BETWEEN_REQUESTS = 5 # seconds
 
 # Make the keyedcache startup warnings go away
 CACHE_TIMEOUT = 0

From fdecaef7eff4e4b4e83cf7ae840c80100bad1370 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Fri, 24 Aug 2012 08:23:46 -0400
Subject: [PATCH 006/181] Fix datetime name conflict

---
 common/lib/xmodule/xmodule/capa_module.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py
index 8f570bcf6c..33a2f6b4c4 100644
--- a/common/lib/xmodule/xmodule/capa_module.py
+++ b/common/lib/xmodule/xmodule/capa_module.py
@@ -7,7 +7,7 @@ import traceback
 import re
 import sys
 
-from datetime import datetime, timedelta
+from datetime import timedelta
 from django.conf import settings
 from lxml import etree
 from pkg_resources import resource_string
@@ -465,10 +465,11 @@ class CapaModule(XModule):
 
         # Problem queued. Students must wait XQUEUE_WAITTIME_BETWEEN_REQUESTS
         if self.lcp.is_queued():
-            current_time = datetime.now()
+            current_time = datetime.datetime.now()
             prev_submit_time = self.lcp.get_recentmost_queuetime() 
-            if (current_time-prev_submit_time).total_seconds() < settings.XQUEUE_WAITTIME_BETWEEN_REQUESTS:
-                msg = 'You must wait %d seconds between queueing requests' % settings.XQUEUE_WAITTIME_BETWEEN_REQUESTS
+            waittime_between_requests = settings.XQUEUE_WAITTIME_BETWEEN_REQUESTS
+            if (current_time-prev_submit_time).total_seconds() < waittime_between_requests:
+                msg = 'You must wait at least %d seconds between queue submissions' % waittime_between_requests
                 return {'success': msg, 'html': ''}
 
         try:

From 97ab53e786c03453a69aeaa1cd7e4462ae37a9dd Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Fri, 24 Aug 2012 08:28:47 -0400
Subject: [PATCH 007/181] Queuekey --> Queuestate in tests

---
 common/lib/xmodule/xmodule/tests/__init__.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/common/lib/xmodule/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py
index 2a380bb8be..5d70056d2a 100644
--- a/common/lib/xmodule/xmodule/tests/__init__.py
+++ b/common/lib/xmodule/xmodule/tests/__init__.py
@@ -292,7 +292,9 @@ class CodeResponseTest(unittest.TestCase):
         answer_ids = sorted(test_lcp.get_question_answers().keys())
         numAnswers = len(answer_ids)
         for i in range(numAnswers):
-            old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuekey=1000 + i))
+            queuekey = 1000 + i
+            queuestate = (queuekey, '')
+            old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
 
         # TODO: Message format inherited from ExternalResponse
         #correct_score_msg = "<edxgrade><awarddetail>EXACT_ANS</awarddetail><message>MESSAGE</message></edxgrade>"
@@ -326,7 +328,7 @@ class CodeResponseTest(unittest.TestCase):
                 new_cmap = CorrectMap()
                 new_cmap.update(old_cmap)
                 npoints = 1 if correctness=='correct' else 0
-                new_cmap.set(answer_id=answer_ids[i], npoints=npoints, correctness=correctness, msg='MESSAGE', queuekey=None)
+                new_cmap.set(answer_id=answer_ids[i], npoints=npoints, correctness=correctness, msg='MESSAGE', queuestate=None)
 
                 test_lcp.update_score(xserver_msgs[correctness], queuekey=1000 + i)
                 self.assertEquals(test_lcp.correct_map.get_dict(), new_cmap.get_dict())

From 088d204d9cbd0d757f66f25be982ba4ba1d8ca3a Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Fri, 24 Aug 2012 11:09:27 -0400
Subject: [PATCH 008/181] Added CodeResponse tests

---
 common/lib/capa/capa/responsetypes.py         |   2 +-
 common/lib/xmodule/xmodule/tests/__init__.py  | 100 ++++++++++++++---
 .../xmodule/tests/test_files/coderesponse.xml |  90 ++--------------
 .../coderesponse_externalresponseformat.xml   | 101 ++++++++++++++++++
 lms/envs/test.py                              |   2 +-
 5 files changed, 199 insertions(+), 96 deletions(-)
 create mode 100644 common/lib/xmodule/xmodule/tests/test_files/coderesponse_externalresponseformat.xml

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index 27801ad871..5cf45adce2 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -1202,7 +1202,7 @@ class CodeResponse(LoncapaResponse):
         '''
          Grader reply is a JSON-dump of the following dict
            { 'correct': True/False,
-             'score': # TODO -- Partial grading
+             'score': Numeric value (floating point is okay) to assign to answer
              'msg': grader_msg }
 
         Returns (valid_score_msg, correct, score, msg):
diff --git a/common/lib/xmodule/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py
index 5d70056d2a..bd5d35c75f 100644
--- a/common/lib/xmodule/xmodule/tests/__init__.py
+++ b/common/lib/xmodule/xmodule/tests/__init__.py
@@ -19,6 +19,7 @@ import capa.calc as calc
 import capa.capa_problem as lcp
 from capa.correctmap import CorrectMap
 from capa.util import convert_files_to_filenames
+from datetime import datetime
 from xmodule import graders, x_module
 from xmodule.x_module import ModuleSystem
 from xmodule.graders import Score, aggregate_scores
@@ -283,30 +284,61 @@ class CodeResponseTest(unittest.TestCase):
     '''
     Test CodeResponse
     '''
+    @staticmethod
+    def make_queuestate(key, time):
+        timestr = datetime.strftime(time,'%Y%m%d%H%M%S')
+        return (key, timestr)
+
+    def test_is_queued(self):
+        ''' 
+        Simple test of whether LoncapaProblem knows when it's been queued
+        '''
+        problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
+        test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs)
+        
+        answer_ids = sorted(test_lcp.get_question_answers().keys())
+        num_answers = len(answer_ids)
+
+        # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
+        cmap = CorrectMap()
+        for i in range(num_answers):
+            cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=None))
+        test_lcp.correct_map.update(cmap)
+
+        self.assertEquals(test_lcp.is_queued(), False)
+
+        # Now we queue the LCP
+        cmap = CorrectMap()
+        for i in range(num_answers):
+            queuestate = CodeResponseTest.make_queuestate(i, datetime.now())
+            cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
+        test_lcp.correct_map.update(cmap)
+
+        self.assertEquals(test_lcp.is_queued(), True)
+
     def test_update_score(self):
+        '''
+        Test whether LoncapaProblem.update_score can deliver queued result to the right subproblem
+        '''
         problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
         test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs)
 
-        # CodeResponse requires internal CorrectMap state. Build it now in the 'queued' state
-        old_cmap = CorrectMap()
         answer_ids = sorted(test_lcp.get_question_answers().keys())
-        numAnswers = len(answer_ids)
-        for i in range(numAnswers):
+        num_answers = len(answer_ids)
+
+        # CodeResponse requires internal CorrectMap state. Build it now in the queued state
+        old_cmap = CorrectMap()
+        for i in range(num_answers):
             queuekey = 1000 + i
-            queuestate = (queuekey, '')
+            queuestate = CodeResponseTest.make_queuestate(1000+i, datetime.now())
             old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
 
-        # TODO: Message format inherited from ExternalResponse
-        #correct_score_msg = "<edxgrade><awarddetail>EXACT_ANS</awarddetail><message>MESSAGE</message></edxgrade>"
-        #incorrect_score_msg = "<edxgrade><awarddetail>WRONG_FORMAT</awarddetail><message>MESSAGE</message></edxgrade>"
-
-        # New message format common to external graders
+        # Message format common to external graders
         correct_score_msg = json.dumps({'correct':True, 'score':1, 'msg':'MESSAGE'})
         incorrect_score_msg = json.dumps({'correct':False, 'score':0, 'msg':'MESSAGE'})
 
         xserver_msgs = {'correct': correct_score_msg,
-                        'incorrect': incorrect_score_msg,
-                        }
+                        'incorrect': incorrect_score_msg,}
 
         # Incorrect queuekey, state should not be updated
         for correctness in ['correct', 'incorrect']:
@@ -316,12 +348,12 @@ class CodeResponseTest(unittest.TestCase):
             test_lcp.update_score(xserver_msgs[correctness], queuekey=0)
             self.assertEquals(test_lcp.correct_map.get_dict(), old_cmap.get_dict())  # Deep comparison
 
-            for i in range(numAnswers):
+            for i in range(num_answers):
                 self.assertTrue(test_lcp.correct_map.is_queued(answer_ids[i]))  # Should be still queued, since message undelivered
 
         # Correct queuekey, state should be updated
         for correctness in ['correct', 'incorrect']:
-            for i in range(numAnswers):  # Target specific answer_id's
+            for i in range(num_answers):  # Target specific answer_id's
                 test_lcp.correct_map = CorrectMap()
                 test_lcp.correct_map.update(old_cmap)
 
@@ -333,13 +365,51 @@ class CodeResponseTest(unittest.TestCase):
                 test_lcp.update_score(xserver_msgs[correctness], queuekey=1000 + i)
                 self.assertEquals(test_lcp.correct_map.get_dict(), new_cmap.get_dict())
 
-                for j in range(numAnswers):
+                for j in range(num_answers):
                     if j == i:
                         self.assertFalse(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be dequeued, message delivered
                     else:
                         self.assertTrue(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be queued, message undelivered
+    
+
+    def test_recentmost_queuetime(self):
+        '''
+        Test whether the LoncapaProblem knows about the time of queue requests
+        '''
+        problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
+        test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs)
+
+        answer_ids = sorted(test_lcp.get_question_answers().keys())
+        num_answers = len(answer_ids)
+
+        # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
+        cmap = CorrectMap()
+        for i in range(num_answers):
+            cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=None))
+        test_lcp.correct_map.update(cmap)
+        
+        self.assertEquals(test_lcp.get_recentmost_queuetime(), None)
+
+        # CodeResponse requires internal CorrectMap state. Build it now in the queued state
+        cmap = CorrectMap()
+        answer_ids = sorted(test_lcp.get_question_answers().keys())
+        num_answers = len(answer_ids)
+        for i in range(num_answers):
+            queuekey = 1000 + i
+            latest_timestamp = datetime.now()
+            queuestate = CodeResponseTest.make_queuestate(1000+i, latest_timestamp)
+            cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
+        test_lcp.correct_map.update(cmap)
+
+        # Queue state only tracks up to second
+        latest_timestamp = datetime.strptime(datetime.strftime(latest_timestamp,'%Y%m%d%H%M%S'),'%Y%m%d%H%M%S')
+
+        self.assertEquals(test_lcp.get_recentmost_queuetime(), latest_timestamp)
 
     def test_convert_files_to_filenames(self):
+        '''
+        Test whether file objects are converted to filenames without altering other structures
+        '''
         problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
         fp = open(problem_file)
         answers_with_file = {'1_2_1': 'String-based answer',
diff --git a/common/lib/xmodule/xmodule/tests/test_files/coderesponse.xml b/common/lib/xmodule/xmodule/tests/test_files/coderesponse.xml
index 42b6e0a54a..1c0bf8d4e6 100644
--- a/common/lib/xmodule/xmodule/tests/test_files/coderesponse.xml
+++ b/common/lib/xmodule/xmodule/tests/test_files/coderesponse.xml
@@ -9,91 +9,23 @@
 Write a program to compute the square of a number
   <coderesponse tests="repeat:2,generate">
     <textbox rows="10" cols="70"  mode="python"/>
-    <answer><![CDATA[
-initial_display = """
-def square(n):
-"""
-
-answer = """
-def square(n):
-    return n**2    
-"""
-
-preamble = """ 
-import sys, time
-"""
-
-test_program = """
-import random
-import operator
-
-def testSquare(n = None):
-    if n is None:
-       n = random.randint(2, 20)
-    print 'Test is: square(%d)'%n
-    return str(square(n))
-
-def main():
-   f = os.fdopen(3,'w')
-   test = int(sys.argv[1])
-   rndlist = map(int,os.getenv('rndlist').split(','))
-   random.seed(rndlist[0])
-   if test == 1: f.write(testSquare(0))
-   elif test == 2: f.write(testSquare(1))
-   else:  f.write(testSquare())
-   f.close()
-
-main()
-sys.exit(0)
-"""
-]]>
-    </answer>
+    <codeparam>
+      <initial_display>def square(x):</initial_display>
+      <answer_display>answer</answer_display>
+      <grader_payload>grader stuff</grader_payload>
+    </codeparam>
   </coderesponse>
 </text>
 
 <text>
-Write a program to compute the cube of a number
+Write a program to compute the square of a number
   <coderesponse tests="repeat:2,generate">
     <textbox rows="10" cols="70"  mode="python"/>
-    <answer><![CDATA[
-initial_display = """
-def cube(n):
-"""
-
-answer = """
-def cube(n):
-    return n**3    
-"""
-
-preamble = """ 
-import sys, time
-"""
-
-test_program = """
-import random
-import operator
-
-def testCube(n = None):
-    if n is None:
-       n = random.randint(2, 20)
-    print 'Test is: cube(%d)'%n
-    return str(cube(n))
-
-def main():
-   f = os.fdopen(3,'w')
-   test = int(sys.argv[1])
-   rndlist = map(int,os.getenv('rndlist').split(','))
-   random.seed(rndlist[0])
-   if test == 1: f.write(testCube(0))
-   elif test == 2: f.write(testCube(1))
-   else:  f.write(testCube())
-   f.close()
-
-main()
-sys.exit(0)
-"""
-]]>
-    </answer>
+    <codeparam>
+      <initial_display>def square(x):</initial_display>
+      <answer_display>answer</answer_display>
+      <grader_payload>grader stuff</grader_payload>
+    </codeparam>
   </coderesponse>
 </text>
 
diff --git a/common/lib/xmodule/xmodule/tests/test_files/coderesponse_externalresponseformat.xml b/common/lib/xmodule/xmodule/tests/test_files/coderesponse_externalresponseformat.xml
new file mode 100644
index 0000000000..42b6e0a54a
--- /dev/null
+++ b/common/lib/xmodule/xmodule/tests/test_files/coderesponse_externalresponseformat.xml
@@ -0,0 +1,101 @@
+<problem>
+<text>
+<h2>Code response</h2>
+
+<p>
+</p>
+
+<text>
+Write a program to compute the square of a number
+  <coderesponse tests="repeat:2,generate">
+    <textbox rows="10" cols="70"  mode="python"/>
+    <answer><![CDATA[
+initial_display = """
+def square(n):
+"""
+
+answer = """
+def square(n):
+    return n**2    
+"""
+
+preamble = """ 
+import sys, time
+"""
+
+test_program = """
+import random
+import operator
+
+def testSquare(n = None):
+    if n is None:
+       n = random.randint(2, 20)
+    print 'Test is: square(%d)'%n
+    return str(square(n))
+
+def main():
+   f = os.fdopen(3,'w')
+   test = int(sys.argv[1])
+   rndlist = map(int,os.getenv('rndlist').split(','))
+   random.seed(rndlist[0])
+   if test == 1: f.write(testSquare(0))
+   elif test == 2: f.write(testSquare(1))
+   else:  f.write(testSquare())
+   f.close()
+
+main()
+sys.exit(0)
+"""
+]]>
+    </answer>
+  </coderesponse>
+</text>
+
+<text>
+Write a program to compute the cube of a number
+  <coderesponse tests="repeat:2,generate">
+    <textbox rows="10" cols="70"  mode="python"/>
+    <answer><![CDATA[
+initial_display = """
+def cube(n):
+"""
+
+answer = """
+def cube(n):
+    return n**3    
+"""
+
+preamble = """ 
+import sys, time
+"""
+
+test_program = """
+import random
+import operator
+
+def testCube(n = None):
+    if n is None:
+       n = random.randint(2, 20)
+    print 'Test is: cube(%d)'%n
+    return str(cube(n))
+
+def main():
+   f = os.fdopen(3,'w')
+   test = int(sys.argv[1])
+   rndlist = map(int,os.getenv('rndlist').split(','))
+   random.seed(rndlist[0])
+   if test == 1: f.write(testCube(0))
+   elif test == 2: f.write(testCube(1))
+   else:  f.write(testCube())
+   f.close()
+
+main()
+sys.exit(0)
+"""
+]]>
+    </answer>
+  </coderesponse>
+</text>
+
+</text>
+</problem>
diff --git a/lms/envs/test.py b/lms/envs/test.py
index 1ab3f550b8..c164889d79 100644
--- a/lms/envs/test.py
+++ b/lms/envs/test.py
@@ -58,7 +58,7 @@ XQUEUE_INTERFACE = {
     },
     "basic_auth": ('anant', 'agarwal'),
 }
-
+XQUEUE_WAITTIME_BETWEEN_REQUESTS = 5 # seconds
 
 # TODO (cpennington): We need to figure out how envs/test.py can inject things
 # into common.py so that we don't have to repeat this sort of thing

From 3787307d93eeb04a37003490c7aa260782fb0477 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Fri, 24 Aug 2012 11:23:26 -0400
Subject: [PATCH 009/181] Put WAITTIME_BETWEEN_REQUESTS in common rather than
 dev env

---
 lms/envs/common.py | 3 +++
 lms/envs/dev.py    | 1 -
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/lms/envs/common.py b/lms/envs/common.py
index 938c4036ae..ce08bf9666 100644
--- a/lms/envs/common.py
+++ b/lms/envs/common.py
@@ -86,6 +86,9 @@ DEFAULT_GROUPS = []
 # If this is true, random scores will be generated for the purpose of debugging the profile graphs
 GENERATE_PROFILE_SCORES = False
 
+# Used with XQueue
+XQUEUE_WAITTIME_BETWEEN_REQUESTS = 5 # seconds
+
 ############################# SET PATH INFORMATION #############################
 PROJECT_ROOT = path(__file__).abspath().dirname().dirname()  # /mitx/lms
 REPO_ROOT = PROJECT_ROOT.dirname()
diff --git a/lms/envs/dev.py b/lms/envs/dev.py
index fe12ee3322..a76e6de262 100644
--- a/lms/envs/dev.py
+++ b/lms/envs/dev.py
@@ -63,7 +63,6 @@ XQUEUE_INTERFACE = {
     },
     "basic_auth": ('anant', 'agarwal'),
 }
-XQUEUE_WAITTIME_BETWEEN_REQUESTS = 5 # seconds
 
 # Make the keyedcache startup warnings go away
 CACHE_TIMEOUT = 0

From 2cf612051c0cbba063527217c6dd9369b335a62a Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Fri, 24 Aug 2012 13:53:32 -0400
Subject: [PATCH 010/181] turn subdomain listings off by default in dev env

---
 lms/envs/dev.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lms/envs/dev.py b/lms/envs/dev.py
index a76e6de262..97803cb47d 100644
--- a/lms/envs/dev.py
+++ b/lms/envs/dev.py
@@ -15,7 +15,7 @@ TEMPLATE_DEBUG = True
 
 MITX_FEATURES['DISABLE_START_DATES'] = True
 MITX_FEATURES['ENABLE_SQL_TRACKING_LOGS'] = True
-MITX_FEATURES['SUBDOMAIN_COURSE_LISTINGS'] = True
+MITX_FEATURES['SUBDOMAIN_COURSE_LISTINGS'] = False  # Enable to test subdomains--otherwise, want all courses to show up
 MITX_FEATURES['SUBDOMAIN_BRANDING'] = True
 
 WIKI_ENABLED = True
@@ -107,7 +107,7 @@ LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1']
 MITX_FEATURES['AUTH_USE_OPENID'] = True
 MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True
 
-INSTALLED_APPS += ('external_auth',) 
+INSTALLED_APPS += ('external_auth',)
 INSTALLED_APPS += ('django_openid_auth',)
 
 OPENID_CREATE_USERS = False

From 7a67fdc4ed54cf2f766114b02f7b4c3c5284bca8 Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Fri, 24 Aug 2012 13:54:46 -0400
Subject: [PATCH 011/181] Fix url_name loading

- cleaned up name loading
- clear fallbacks, including hashing content if no name specified
- log errors in error tracker for content devs to see
---
 common/lib/xmodule/xmodule/modulestore/xml.py | 90 ++++++++++++++-----
 1 file changed, 66 insertions(+), 24 deletions(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py
index 3eca72987e..98260f5982 100644
--- a/common/lib/xmodule/xmodule/modulestore/xml.py
+++ b/common/lib/xmodule/xmodule/modulestore/xml.py
@@ -1,3 +1,4 @@
+import hashlib
 import json
 import logging
 import os
@@ -43,14 +44,76 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
 
         xmlstore: the XMLModuleStore to store the loaded modules in
         """
-        self.unnamed_modules = 0
-        self.used_slugs = set()
+        self.unnamed = defaultdict(int)     # category -> num of new url_names for that category
+        self.used_names = defaultdict(set)  # category -> set of used url_names
         self.org, self.course, self.url_name = course_id.split('/')
 
         def process_xml(xml):
             """Takes an xml string, and returns a XModuleDescriptor created from
             that xml.
             """
+
+            def make_name_unique(xml_data):
+                """
+                Make sure that the url_name of xml_data is unique.  If a previously loaded
+                unnamed descriptor stole this element's url_name, create a new one.
+
+                Removes 'slug' attribute if present, and adds or overwrites the 'url_name' attribute.
+                """
+                # VS[compat]. Take this out once course conversion is done (perhaps leave the uniqueness check)
+
+                attr = xml_data.attrib
+                tag = xml_data.tag
+                id = lambda x: x
+                # Things to try to get a name, in order  (key, cleaning function, remove key after reading?)
+                lookups = [('url_name', id, False),
+                           ('slug', id, True),
+                           ('name', Location.clean, False),
+                           ('display_name', Location.clean, False)]
+
+                url_name = None
+                for key, clean, remove in lookups:
+                    if key in attr:
+                        url_name = clean(attr[key])
+                        if remove:
+                            del attr[key]
+                        break
+
+                def fallback_name():
+                    """Return the fallback name for this module.  This is a function instead of a variable
+                    because we want it to be lazy."""
+                    return tag + "_" + hashlib.sha1(xml).hexdigest()[:12]
+
+                # Fallback if there was nothing we could use:
+                if url_name is None or url_name == "":
+                    # use the hash of the content--the first 12 bytes should be plenty.
+                    url_name = fallback_name()
+                    # Don't log a warning--we don't need this in the log.  Do
+                    # put it in the error tracker--content folks need to see it.
+                    need_uniq_names = ('problem', 'sequence', 'video', 'course', 'chapter')
+
+                    if tag in need_uniq_names:
+                        error_tracker("ERROR: no name of any kind specified for {tag}.  Student "
+                                      "state won't work right.  Problem xml: '{xml}...'".format(tag=tag, xml=xml[:100]))
+                    else:
+                        # TODO (vshnayder): We may want to enable this once course repos are cleaned up.
+                        # (or we may want to give up on the requirement for non-state-relevant issues...)
+                        #error_tracker("WARNING: no name specified for module. xml='{0}...'".format(xml[:100]))
+                        pass
+
+                # Make sure everything is unique
+                if url_name in self.used_names[tag]:
+                    msg = ("Non-unique url_name in xml.  This may break content.  url_name={0}.  Content={1}"
+                                .format(url_name, xml[:100]))
+                    error_tracker("ERROR: " + msg)
+                    log.warning(msg)
+                    # Just set name to fallback_name--if there are multiple things with the same fallback name,
+                    # they are actually identical, so it's fragile, but not immediately broken.
+                    url_name = fallback_name()
+
+                self.used_names[tag].add(url_name)
+                xml_data.set('url_name', url_name)
+
             try:
                 # VS[compat]
                 # TODO (cpennington): Remove this once all fall 2012 courses
@@ -62,32 +125,11 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                     err=str(err), xml=xml))
                 raise
 
-            # VS[compat]. Take this out once course conversion is done
-            if xml_data.get('slug') is None and xml_data.get('url_name') is None:
-                if xml_data.get('name'):
-                    slug = Location.clean(xml_data.get('name'))
-                elif xml_data.get('display_name'):
-                    slug = Location.clean(xml_data.get('display_name'))
-                else:
-                    self.unnamed_modules += 1
-                    slug = '{tag}_{count}'.format(tag=xml_data.tag,
-                                                  count=self.unnamed_modules)
-
-                while slug in self.used_slugs:
-                    self.unnamed_modules += 1
-                    slug = '{slug}_{count}'.format(slug=slug,
-                                                   count=self.unnamed_modules)
-
-                self.used_slugs.add(slug)
-                # log.debug('-> slug=%s' % slug)
-                xml_data.set('url_name', slug)
+            make_name_unique(xml_data)
 
             descriptor = XModuleDescriptor.load_from_xml(
                 etree.tostring(xml_data), self, self.org,
                 self.course, xmlstore.default_class)
-
-            #log.debug('==> importing descriptor location %s' %
-            #          repr(descriptor.location))
             descriptor.metadata['data_dir'] = course_dir
 
             xmlstore.modules[course_id][descriptor.location] = descriptor

From dc52956c28d7b112cc4711994ad35a23bb939259 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Fri, 24 Aug 2012 14:09:57 -0400
Subject: [PATCH 012/181] don't list all users if too many

---
 .../management/commands/manage_course_groups.py       | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/lms/djangoapps/lms_migration/management/commands/manage_course_groups.py b/lms/djangoapps/lms_migration/management/commands/manage_course_groups.py
index 0043b483e0..f3a39db5ca 100644
--- a/lms/djangoapps/lms_migration/management/commands/manage_course_groups.py
+++ b/lms/djangoapps/lms_migration/management/commands/manage_course_groups.py
@@ -36,9 +36,14 @@ class Command(BaseCommand):
         # users in group
         
         uall = User.objects.all()
-        print "----"
-        print "List of All Users: %s" % [str(x.username) for x in uall]
-        print "----"
+        if uall.count()<50:
+            print "----"
+            print "List of All Users: %s" % [str(x.username) for x in uall]
+            print "----"
+        else:
+            print "----"
+            print "There are %d users, which is too many to list" % uall.count()
+            print "----"
         
         while True:
         

From d1760f9a22a075648190003f9647bc64ba2cbfb0 Mon Sep 17 00:00:00 2001
From: Tom Giannattasio <tom@mitx.mit.edu>
Date: Fri, 24 Aug 2012 16:14:04 -0400
Subject: [PATCH 013/181] styled markdown modal and link

---
 lms/static/sass/course/wiki/_wiki.scss      | 78 ++++++++++++++---
 lms/templates/wiki/base.html                |  5 +-
 lms/templates/wiki/includes/cheatsheet.html | 95 +++++++++++----------
 3 files changed, 117 insertions(+), 61 deletions(-)

diff --git a/lms/static/sass/course/wiki/_wiki.scss b/lms/static/sass/course/wiki/_wiki.scss
index ead58bf6a6..912b416ff6 100644
--- a/lms/static/sass/course/wiki/_wiki.scss
+++ b/lms/static/sass/course/wiki/_wiki.scss
@@ -567,29 +567,79 @@ section.wiki {
     background: #f00 !important;
   }
 
+  .cheatsheet {
+    float: right;
+    position: relative;
+    top: -26px;
+    font-size: 12px;
+  }
+
   #cheatsheetLink {
-   text-align:right;
-   display: float;
+    text-align: right;
+    display: float;
   }
 
   #cheatsheetModal {
-   width: 350px;
-   margin-left: 100px;
-   margin-top: -100px;
-   }
+    width: 950px;
+    margin-left: -450px;
+    margin-top: -100px;
+
+    .left-column {
+      margin-right: 10px;
+    }
+
+    .left-column,
+    .right-column {
+      float: left;
+      width: 450px;
+    }
+
+    .close-btn {
+      display: block;
+      position: absolute;
+      top: -8px;
+      right: -8px;
+      width: 30px;
+      height: 30px;
+      border-radius: 30px;
+      border: 1px solid #ccc;
+      @include linear-gradient(top, #eee, #d2d2d2);
+      font-size: 22px;
+      line-height: 28px;    
+      color: #333;
+      text-align: center;
+      @include box-shadow(0 1px 0 #fff inset, 0 1px 2px rgba(0, 0, 0, .2));
+    }
+  }
 
   #cheatsheet-body {
-   background: #FFF;
-   text-align: left;
-   padding: 10px;
+    background: #fff;
+    text-align: left;
+    padding: 20px;
+    font-size: 14px;
+    @include clearfix;
+
+    h3 {
+      font-weight: bold;
+    }
+
+    ul {
+      list-style: circle;
+      line-height: 1.4;
+      color: #333;
+    }
+  }
+
+  #cheatsheet-body section + section {
+    margin-top: 40px;
   }
 
   #cheatsheet-body pre{
-   color: #000;
-   text-align: left;
-   background: #EEE;
-   margin:10px;
-   padding: 10px;
+    color: #000;
+    text-align: left;
+    background: #eee;
+    padding: 10px;
+    font-size: 12px;
   }
 
   /*-----------------
diff --git a/lms/templates/wiki/base.html b/lms/templates/wiki/base.html
index a346be0e3e..5c7381ac05 100644
--- a/lms/templates/wiki/base.html
+++ b/lms/templates/wiki/base.html
@@ -42,9 +42,12 @@
     <script type="text/javascript">
 	    $(document).ready(function () {
           editor = $('#div_id_content div.controls');
-          cs = editor.prepend('<a id="cheatsheetLink" href="#">cheatsheet</a>');
+          cs = editor.prepend('<p class="cheatsheet">Markdown syntax is allowed. See the <a id="cheatsheetLink" href="#">cheatsheet</a> for help.</p>');
           cs.click(function() {
               $('#cheatsheetModal').modal('show');
+          });
+          $('#cheatsheetModal .close-btn').click(function(e) {
+            $('#cheatsheetModal').modal('hide');
           });
 	    });
     </script>
diff --git a/lms/templates/wiki/includes/cheatsheet.html b/lms/templates/wiki/includes/cheatsheet.html
index d58920b814..429c1ea22e 100644
--- a/lms/templates/wiki/includes/cheatsheet.html
+++ b/lms/templates/wiki/includes/cheatsheet.html
@@ -1,53 +1,56 @@
-<section id="cheatsheetModal" class="modal hide fade" >
-<div id="cheatsheet-body" class="modal-body">
-<h2>Wiki Syntax Help</h2>
-<p> This wiki uses <strong>Markdown</strong> for styling.
-  There are several <a href="http://daringfireball.net/projects/markdown/basics" target="_blank">useful</a>
-  <a href="http://greg.vario.us/doc/markdown.txt" target="_blank">guides</a>
-  <a href="http://www.lowendtalk.com/discussion/6/miniature-markdown-guide" target="_blank">online</a>.
-<p> To create a new wiki article, create a link to it. Clicking the link gives you the creation page.
-<pre>
-[Article Name](wiki:ArticleName)
-</pre>
-<h2> Useful examples:</h2>
-<pre>
+<section id="cheatsheetModal" class="modal hide fade">
+	<a href="#" class="close-btn">×</a>
+	<div id="cheatsheet-body" class="modal-body">
+		<div class="left-column">
+			<section>
+				<h2>Wiki Syntax Help</h2>
+				<p>This wiki uses <strong>Markdown</strong> for styling. There are several useful guides online. See any of the links below for in-depth details:</p>
+				<ul>
+					<li><a href="http://daringfireball.net/projects/markdown/basics" target="_blank">Markdown: Basics</a></li>
+					<li><a href="http://greg.vario.us/doc/markdown.txt" target="_blank">Quick Markdown Syntax Guide</a></li>
+					<li><a href="http://www.lowendtalk.com/discussion/6/miniature-markdown-guide" target="_blank">Miniature Markdown Guide</a></li>
+				</ul>
+				<p>To create a new wiki article, create a link to it. Clicking the link gives you the creation page.</p>
+				<pre>[Article Name](wiki:ArticleName)</pre>
+			</section>
+			<section>
+				<h3>edX Additions:</h3>
+				<pre>
+circuit-schematic:</pre>
+				<pre>
+$LaTeX Math Expression$</pre>
+			</section>
+		</div>
+
+		<div class="right-column">
+			<section>
+				<h3>Useful examples:</h3>
+				<pre>
 http://wikipedia.org
 [Wikipedia](http://wikipedia.org)
-[edX Wiki](wiki:/edx/)
-</pre>
-<pre>
+[edX Wiki](wiki:/edx/)</pre>
+				<pre>
 Huge Header
-===========
-</pre>
-<pre>
+===========</pre>
+				<pre>
 Smaller Header
---------------
-</pre>
-<pre>
-*emphasis* or _emphasis_
-</pre>
-<pre>
-**strong** or __strong__
-</pre>
-<pre>
+--------------</pre>
+				<pre>
+*emphasis* or _emphasis_</pre>
+				<pre>
+**strong** or __strong__</pre>
+				<pre>
 - Unordered List
-  - Sub Item 1
-  - Sub Item 2
-</pre>
-<pre>
+	- Sub Item 1
+	- Sub Item 2</pre>
+				<pre>
 1. Ordered
-2. List
-</pre>
-<pre>
-> Quotes
-</pre>
-<h2>edX Additions:</h2>
-<pre>
-circuit-schematic:
-</pre>
-<pre>
-$LaTeX Math Expression$
-</pre>
-</div>
-</div>
+2. List</pre>
+				<pre>
+> Quotes</pre>
+			</section>
+		</div>
+
+		</div>
+	</div>
 </section>

From 45a64a9bd4cc33968c0cf5890eb46e5285f4e36f Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Fri, 24 Aug 2012 16:33:05 -0400
Subject: [PATCH 014/181] Hide queue length information from the student

---
 common/lib/capa/capa/inputtypes.py          | 2 +-
 common/lib/capa/capa/templates/textbox.html | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/common/lib/capa/capa/inputtypes.py b/common/lib/capa/capa/inputtypes.py
index ad54736359..9290503c9d 100644
--- a/common/lib/capa/capa/inputtypes.py
+++ b/common/lib/capa/capa/inputtypes.py
@@ -384,7 +384,7 @@ def textbox(element, value, status, render_template, msg=''):
     if status == 'incomplete': # Flag indicating that the problem has been queued, 'msg' is length of queue
         status = 'queued'
         queue_len = msg
-        msg = 'Submitted to grader. (Queue length: %s)' % queue_len
+        msg = 'Submitted to grader.'
 
     # For CodeMirror
     mode = element.get('mode','python')
diff --git a/common/lib/capa/capa/templates/textbox.html b/common/lib/capa/capa/templates/textbox.html
index 647f4fe4e8..19c43482a8 100644
--- a/common/lib/capa/capa/templates/textbox.html
+++ b/common/lib/capa/capa/templates/textbox.html
@@ -21,7 +21,7 @@
   <div style="display:none;" name="${hidden}" inputid="input_${id}" />
   % endif
   <br/>
-  <span class="debug">(${state})</span>
+  <span style="display:none;" class="debug">(${state})</span>
   <br/>
   <span class="message">${msg|n}</span>
   <br/>

From 1dcaf21f8131edb7f0de95b51fac47ac999c8303 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Sat, 25 Aug 2012 07:57:17 -0400
Subject: [PATCH 015/181] Send idhash

---
 common/lib/capa/capa/responsetypes.py      | 10 +++++++++-
 common/lib/capa/capa/xqueue_interface.py   |  7 ++-----
 lms/djangoapps/courseware/module_render.py |  4 +++-
 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index b2d56b48ca..032970e324 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -16,6 +16,7 @@ import numpy
 import random
 import re
 import requests
+import time
 import traceback
 import hashlib
 import abc
@@ -1124,7 +1125,9 @@ class CodeResponse(LoncapaResponse):
         qinterface = self.system.xqueue['interface']
 
         # Generate header
-        queuekey = xqueue_interface.make_hashkey(str(self.system.seed)+self.answer_id)
+        queuekey = xqueue_interface.make_hashkey(str(self.system.seed) + str(time.time()) +
+                                                 str(self.system.xqueue['student_identifier']) + 
+                                                 self.answer_id)
         xheader = xqueue_interface.make_xheader(lms_callback_url=self.system.xqueue['callback_url'],
                                                 lms_key=queuekey,
                                                 queue_name=self.queue_name)
@@ -1137,6 +1140,11 @@ class CodeResponse(LoncapaResponse):
 
         contents = self.payload.copy() 
 
+        # Anonymized student identifier to the external grader
+        student_identifier_hash = xqueue_interface.make_hashkey(self.system.xqueue['student_identifier'])
+        student_info = {'student_identifier_hash': student_identifier_hash}
+        contents.update({'student_info': json.dumps(student_info)})
+
         # Submit request. When successful, 'msg' is the prior length of the queue
         if is_list_of_files(submission):
             contents.update({'student_response': ''}) # TODO: Is there any information we want to send here?
diff --git a/common/lib/capa/capa/xqueue_interface.py b/common/lib/capa/capa/xqueue_interface.py
index 2930eb682d..0dfdabf8d0 100644
--- a/common/lib/capa/capa/xqueue_interface.py
+++ b/common/lib/capa/capa/xqueue_interface.py
@@ -5,20 +5,17 @@ import hashlib
 import json
 import logging
 import requests
-import time
 
 
 log = logging.getLogger('mitx.' + __name__)
 
 
-def make_hashkey(seed=None):
+def make_hashkey(seed):
     '''
     Generate a string key by hashing 
     '''
     h = hashlib.md5()
-    if seed is not None:
-        h.update(str(seed))
-    h.update(str(time.time()))
+    h.update(str(seed))
     return h.hexdigest()
 
 
diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py
index da9828fb12..467b147d0f 100644
--- a/lms/djangoapps/courseware/module_render.py
+++ b/lms/djangoapps/courseware/module_render.py
@@ -217,7 +217,9 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
 
     xqueue = {'interface': xqueue_interface,
               'callback_url': xqueue_callback_url,
-              'default_queuename': xqueue_default_queuename.replace(' ', '_')}
+              'default_queuename': xqueue_default_queuename.replace(' ', '_'),
+              'student_identifier': user.id,
+             }
 
     def inner_get_module(location):
         """

From 27459efcf7d2fcc25c63a42a523a38aa1de397e5 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Sat, 25 Aug 2012 08:06:17 -0400
Subject: [PATCH 016/181] Update CodeResponse test

---
 common/lib/xmodule/xmodule/tests/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/lib/xmodule/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py
index 2a380bb8be..01462d16ac 100644
--- a/common/lib/xmodule/xmodule/tests/__init__.py
+++ b/common/lib/xmodule/xmodule/tests/__init__.py
@@ -35,7 +35,7 @@ i4xs = ModuleSystem(
     user=Mock(),
     filestore=fs.osfs.OSFS(os.path.dirname(os.path.realpath(__file__))+"/test_files"),
     debug=True,
-    xqueue={'interface':None, 'callback_url':'/', 'default_queuename': 'testqueue'},
+    xqueue={'interface':None, 'callback_url':'/', 'default_queuename': 'testqueue', 'student_identifier': 0},
     node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules")
 )
 

From f7c87aa7a1a318dae587276a489d95e0b2f91b11 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Sat, 25 Aug 2012 09:03:47 -0400
Subject: [PATCH 017/181] Anonymous student id is a property of ModuleSystem

---
 common/lib/capa/capa/responsetypes.py        |  8 ++++----
 common/lib/xmodule/xmodule/tests/__init__.py |  5 +++--
 common/lib/xmodule/xmodule/x_module.py       | 11 +++++++++--
 lms/djangoapps/courseware/module_render.py   | 10 ++++++++--
 4 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index 032970e324..fb7a69a2f2 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -1123,10 +1123,11 @@ class CodeResponse(LoncapaResponse):
         # Prepare xqueue request
         #------------------------------------------------------------ 
         qinterface = self.system.xqueue['interface']
+        anonymous_student_id = self.system.anonymous_student_id
 
         # Generate header
         queuekey = xqueue_interface.make_hashkey(str(self.system.seed) + str(time.time()) +
-                                                 str(self.system.xqueue['student_identifier']) + 
+                                                 anonymous_student_id +  
                                                  self.answer_id)
         xheader = xqueue_interface.make_xheader(lms_callback_url=self.system.xqueue['callback_url'],
                                                 lms_key=queuekey,
@@ -1134,15 +1135,14 @@ class CodeResponse(LoncapaResponse):
 
         # Generate body
         if is_list_of_files(submission):
-            self.context.update({'submission': queuekey}) # For tracking. TODO: May want to record something else here
+            self.context.update({'submission': ''}) # TODO: Get S3 pointer from the Queue
         else:
             self.context.update({'submission': submission})
 
         contents = self.payload.copy() 
 
         # Anonymized student identifier to the external grader
-        student_identifier_hash = xqueue_interface.make_hashkey(self.system.xqueue['student_identifier'])
-        student_info = {'student_identifier_hash': student_identifier_hash}
+        student_info = {'anonymous_student_id': anonymous_student_id}
         contents.update({'student_info': json.dumps(student_info)})
 
         # Submit request. When successful, 'msg' is the prior length of the queue
diff --git a/common/lib/xmodule/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py
index 01462d16ac..0d10184501 100644
--- a/common/lib/xmodule/xmodule/tests/__init__.py
+++ b/common/lib/xmodule/xmodule/tests/__init__.py
@@ -35,8 +35,9 @@ i4xs = ModuleSystem(
     user=Mock(),
     filestore=fs.osfs.OSFS(os.path.dirname(os.path.realpath(__file__))+"/test_files"),
     debug=True,
-    xqueue={'interface':None, 'callback_url':'/', 'default_queuename': 'testqueue', 'student_identifier': 0},
-    node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules")
+    xqueue={'interface':None, 'callback_url':'/', 'default_queuename': 'testqueue'},
+    node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules"),
+    anonymous_student_id = 'student'
 )
 
 
diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py
index c581911c03..0dc16bd976 100644
--- a/common/lib/xmodule/xmodule/x_module.py
+++ b/common/lib/xmodule/xmodule/x_module.py
@@ -717,7 +717,8 @@ class ModuleSystem(object):
                  filestore=None,
                  debug=False,
                  xqueue=None,
-                 node_path=""):
+                 node_path="",
+                 anonymous_student_id=''):
         '''
         Create a closure around the system environment.
 
@@ -742,11 +743,16 @@ class ModuleSystem(object):
                          at settings.DATA_DIR.
 
         xqueue - Dict containing XqueueInterface object, as well as parameters
-                    for the specific StudentModule
+                    for the specific StudentModule:
+                    xqueue = {'interface': XQueueInterface object,
+                              'callback_url': Callback into the LMS,
+                              'queue_name': Target queuename in Xqueue}
 
         replace_urls - TEMPORARY - A function like static_replace.replace_urls
                          that capa_module can use to fix up the static urls in
                          ajax results.
+
+        anonymous_student_id - Used for tracking modules with student id
         '''
         self.ajax_url = ajax_url
         self.xqueue = xqueue
@@ -758,6 +764,7 @@ class ModuleSystem(object):
         self.seed = user.id if user is not None else 0
         self.replace_urls = replace_urls
         self.node_path = node_path
+        self.anonymous_student_id = anonymous_student_id
 
     def get(self, attr):
         '''	provide uniform access to attributes (like etree).'''
diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py
index 467b147d0f..b278d2615b 100644
--- a/lms/djangoapps/courseware/module_render.py
+++ b/lms/djangoapps/courseware/module_render.py
@@ -1,3 +1,4 @@
+import hashlib
 import json
 import logging
 import sys
@@ -173,6 +174,11 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
     if not has_access(user, descriptor, 'load'):
         return None
 
+    # Anonymized student identifier
+    h = hashlib.md5() # TODO: Seed with LMS secret key
+    h.update(str(user.id))
+    anonymous_student_id = h.hexdigest()
+
     #TODO Only check the cache if this module can possibly have state
     instance_module = None
     shared_module = None
@@ -218,7 +224,6 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
     xqueue = {'interface': xqueue_interface,
               'callback_url': xqueue_callback_url,
               'default_queuename': xqueue_default_queuename.replace(' ', '_'),
-              'student_identifier': user.id,
              }
 
     def inner_get_module(location):
@@ -243,7 +248,8 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
                           # a module is coming through get_html and is therefore covered
                           # by the replace_static_urls code below
                           replace_urls=replace_urls,
-                          node_path=settings.NODE_PATH
+                          node_path=settings.NODE_PATH,
+                          anonymous_student_id=anonymous_student_id
                           )
     # pass position specified in URL to module through ModuleSystem
     system.set('position', position)

From 25ea8b29905fde7d1a43e85bb2a0264d4a3ac158 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Sat, 25 Aug 2012 09:18:10 -0400
Subject: [PATCH 018/181] Use LMS secret key to seed anonymizer

---
 lms/djangoapps/courseware/module_render.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py
index b278d2615b..5bf1cc8e97 100644
--- a/lms/djangoapps/courseware/module_render.py
+++ b/lms/djangoapps/courseware/module_render.py
@@ -175,7 +175,8 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
         return None
 
     # Anonymized student identifier
-    h = hashlib.md5() # TODO: Seed with LMS secret key
+    h = hashlib.md5()
+    h.update(settings.SECRET_KEY)
     h.update(str(user.id))
     anonymous_student_id = h.hexdigest()
 

From c406be8c6a28db205bb2417dab1dd912f5475385 Mon Sep 17 00:00:00 2001
From: kimth <kimth@edx.org>
Date: Sun, 26 Aug 2012 13:43:21 +0000
Subject: [PATCH 019/181] Check grader message has proper XML structure

---
 common/lib/capa/capa/responsetypes.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index b2d56b48ca..92c6f62048 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -1165,7 +1165,7 @@ class CodeResponse(LoncapaResponse):
 
         (valid_score_msg, correct, points, msg) = self._parse_score_msg(score_msg) 
         if not valid_score_msg:
-            oldcmap.set(self.answer_id, msg='Error: Invalid grader reply.')
+            oldcmap.set(self.answer_id, msg='Invalid grader reply. Please contact the course staff.')
             return oldcmap
         
         correctness = 'correct' if correct else 'incorrect'
@@ -1203,10 +1203,10 @@ class CodeResponse(LoncapaResponse):
         Returns (valid_score_msg, correct, score, msg):
             valid_score_msg: Flag indicating valid score_msg format (Boolean)
             correct:         Correctness of submission (Boolean)
-            score:           # TODO: Implement partial grading
+            score:           Points to be assigned (numeric, can be float)
             msg:             Message from grader to display to student (string)
         '''
-        fail = (False, False, -1, '')
+        fail = (False, False, 0, '')
         try:
             score_result = json.loads(score_msg)
         except (TypeError, ValueError):
@@ -1216,7 +1216,19 @@ class CodeResponse(LoncapaResponse):
         for tag in ['correct', 'score', 'msg']:
             if not score_result.has_key(tag):
                 return fail
-        return (True, score_result['correct'], score_result['score'], score_result['msg'])
+
+        # Next, we need to check that the contents of the external grader message 
+        #   is safe for the LMS.
+        # 1) Make sure that the message is valid XML (proper opening/closing tags)
+        # 2) TODO: Is the message actually HTML?
+        msg = score_result['msg']
+        try:
+            etree.fromstring(msg)
+        except etree.XMLSyntaxError as err:
+            log.error("Unable to parse external grader message as valid XML: score_msg['msg']=%s" % msg)
+            return fail
+            
+        return (True, score_result['correct'], score_result['score'], msg)
         
 
 #-----------------------------------------------------------------------------

From 79568dfdf68ced51ec739ddcb7da2ea6a89e10d6 Mon Sep 17 00:00:00 2001
From: kimth <kimth@edx.org>
Date: Sun, 26 Aug 2012 17:37:01 +0000
Subject: [PATCH 020/181] Add error logging for external grader messages

---
 common/lib/capa/capa/responsetypes.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index 92c6f62048..e7d7406cbd 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -1210,11 +1210,14 @@ class CodeResponse(LoncapaResponse):
         try:
             score_result = json.loads(score_msg)
         except (TypeError, ValueError):
+            log.error("External grader message should be a JSON-serialized dict")
             return fail
         if not isinstance(score_result, dict):
+            log.error("External grader message should be a JSON-serialized dict")
             return fail
         for tag in ['correct', 'score', 'msg']:
             if not score_result.has_key(tag):
+                log.error("External grader message is missing one or more required tags: 'correct', 'score', 'msg'")
                 return fail
 
         # Next, we need to check that the contents of the external grader message 

From af871d92aa9165668ad920c8b54a8549fb56f652 Mon Sep 17 00:00:00 2001
From: kimth <kimth@edx.org>
Date: Sun, 26 Aug 2012 21:15:34 +0000
Subject: [PATCH 021/181] Send ext grader the time of submission

---
 common/lib/capa/capa/responsetypes.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index fb7a69a2f2..580c4f979a 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -27,6 +27,7 @@ import xml.sax.saxutils as saxutils
 # specific library imports
 from calc import evaluator, UndefinedVariable
 from correctmap import CorrectMap
+from datetime import datetime
 from util import *
 from lxml import etree
 from lxml.html.soupparser import fromstring as fromstring_bs	 # uses Beautiful Soup!!! FIXME?
@@ -1141,8 +1142,11 @@ class CodeResponse(LoncapaResponse):
 
         contents = self.payload.copy() 
 
-        # Anonymized student identifier to the external grader
-        student_info = {'anonymous_student_id': anonymous_student_id}
+        # Metadata related to the student submission revealed to the external grader
+        current_time = datetime.now()
+        student_info = {'anonymous_student_id': anonymous_student_id,
+                        'submission_time': str(current_time),
+                       }
         contents.update({'student_info': json.dumps(student_info)})
 
         # Submit request. When successful, 'msg' is the prior length of the queue

From 9102c7609cc03a499dae45aa43ddc6e25623d96e Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Mon, 27 Aug 2012 10:20:16 -0400
Subject: [PATCH 022/181] Fix latent bug in access checks in get_module

* All access checks now done for the "user" param, ignoring request.user
  - This matters for xqueue callbacks and for instructor view of student progress
* An effect of this change is that if a student couldn't see their own 'progress' tab,
  the course instructor won't be able to see it either
---
 lms/djangoapps/courseware/module_render.py | 10 ++++------
 lms/djangoapps/courseware/tests/tests.py   | 13 +++++++++++--
 lms/djangoapps/courseware/views.py         |  4 ++++
 3 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py
index 8a96d2533f..6c3db9bc18 100644
--- a/lms/djangoapps/courseware/module_render.py
+++ b/lms/djangoapps/courseware/module_render.py
@@ -144,8 +144,8 @@ def get_module(user, request, location, student_module_cache, course_id, positio
 
     Arguments:
       - user                  : User for whom we're getting the module
-      - request               : current django HTTPrequest -- used in particular for auth
-                                (This is important e.g. for prof impersonation of students in progress view)
+      - request               : current django HTTPrequest.  Note: request.user isn't used for anything--all auth
+                                and such works based on user.
       - location              : A Location-like object identifying the module to load
       - student_module_cache  : a StudentModuleCache
       - course_id             : the course_id in the context of which to load module
@@ -171,12 +171,10 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
     descriptor = modulestore().get_instance(course_id, location)
 
     # Short circuit--if the user shouldn't have access, bail without doing any work
-    # NOTE: Do access check on request.user -- that's who actually needs access (e.g. could be prof
-    # impersonating a user)
-    if not has_access(request.user, descriptor, 'load'):
+    if not has_access(user, descriptor, 'load'):
         return None
 
-    #TODO Only check the cache if this module can possibly have state
+    # Only check the cache if this module can possibly have state
     instance_module = None
     shared_module = None
     if user.is_authenticated():
diff --git a/lms/djangoapps/courseware/tests/tests.py b/lms/djangoapps/courseware/tests/tests.py
index bb48dce609..88e0f53f70 100644
--- a/lms/djangoapps/courseware/tests/tests.py
+++ b/lms/djangoapps/courseware/tests/tests.py
@@ -411,8 +411,6 @@ class TestViewAuth(PageLoader):
             """list of urls that only instructors/staff should be able to see"""
             urls = reverse_urls(['instructor_dashboard','gradebook','grade_summary'],
                                 course)
-            urls.append(reverse('student_progress', kwargs={'course_id': course.id,
-                                                     'student_id': user(self.student).id}))
             return urls
 
         def check_non_staff(course):
@@ -435,6 +433,17 @@ class TestViewAuth(PageLoader):
                 print 'checking for 200 on {0}'.format(url)
                 self.check_for_get_code(200, url)
 
+            # The student progress tab is not accessible to a student
+            # before launch, so the instructor view-as-student feature should return a 404 as well.
+            # TODO (vshnayder): If this is not the behavior we want, will need
+            # to make access checking smarter and understand both the effective
+            # user (the student), and the requesting user (the prof)
+            url = reverse('student_progress', kwargs={'course_id': course.id,
+                                                     'student_id': user(self.student).id})
+            print 'checking for 404 on view-as-student: {0}'.format(url)
+            self.check_for_get_code(404, url)
+
+
         # First, try with an enrolled student
         print '=== Testing student access....'
         self.login(self.student, self.password)
diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py
index 50b7a2d645..b532688037 100644
--- a/lms/djangoapps/courseware/views.py
+++ b/lms/djangoapps/courseware/views.py
@@ -333,6 +333,10 @@ def progress(request, course_id, student_id=None):
     course_module = get_module(student, request, course.location,
                                student_module_cache, course_id)
 
+    # The course_module should be accessible, but check anyway just in case something went wrong:
+    if course_module is None:
+        raise Http404("No access to this course")
+
     courseware_summary = grades.progress_summary(student, course_module,
                                                  course.grader, student_module_cache)
     grade_summary = grades.grade(student, request, course, student_module_cache)

From 7e2e90cfa58957dcf1cf41ad96d82310f1935dff Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Mon, 27 Aug 2012 11:29:13 -0400
Subject: [PATCH 023/181] move comment into function re: #549

---
 common/lib/xmodule/xmodule/modulestore/xml.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py
index 98260f5982..92eca8f5e6 100644
--- a/common/lib/xmodule/xmodule/modulestore/xml.py
+++ b/common/lib/xmodule/xmodule/modulestore/xml.py
@@ -82,11 +82,11 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                 def fallback_name():
                     """Return the fallback name for this module.  This is a function instead of a variable
                     because we want it to be lazy."""
+                    # use the hash of the content--the first 12 bytes should be plenty.
                     return tag + "_" + hashlib.sha1(xml).hexdigest()[:12]
 
                 # Fallback if there was nothing we could use:
                 if url_name is None or url_name == "":
-                    # use the hash of the content--the first 12 bytes should be plenty.
                     url_name = fallback_name()
                     # Don't log a warning--we don't need this in the log.  Do
                     # put it in the error tracker--content folks need to see it.

From f81f94ec674305d03ed877e34d1aa024ef922205 Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Wed, 22 Aug 2012 09:59:01 -0400
Subject: [PATCH 024/181] Allow location url_names to contain ':', mapping to /
 on file load

* New format: {tag}://{org}/{course}/{category}/{name}[@{revision}]
* Updated tests, code
* Added test chapter with : in url_name in toy course
* added test html page with : in url_name
* added a note to docs
---
 cms/djangoapps/github_sync/tests/__init__.py  |  2 +-
 common/lib/xmodule/xmodule/html_module.py     | 30 ++++++++++++----
 .../xmodule/xmodule/modulestore/__init__.py   | 36 +++++++++++++------
 .../modulestore/tests/test_location.py        | 32 +++++++++++------
 .../lib/xmodule/xmodule/tests/test_import.py  | 34 ++++++++++++++++++
 common/lib/xmodule/xmodule/xml_module.py      | 16 ++++++---
 common/test/data/toy/chapter/secret/magic.xml |  3 ++
 common/test/data/toy/course/2012_Fall.xml     |  3 +-
 .../data/toy/html/{ => secret}/toylab.html    |  0
 .../data/toy/html/{ => secret}/toylab.xml     |  0
 common/test/data/toy/policies/2012_Fall.json  |  2 +-
 11 files changed, 123 insertions(+), 35 deletions(-)
 create mode 100644 common/test/data/toy/chapter/secret/magic.xml
 rename common/test/data/toy/html/{ => secret}/toylab.html (100%)
 rename common/test/data/toy/html/{ => secret}/toylab.xml (100%)

diff --git a/cms/djangoapps/github_sync/tests/__init__.py b/cms/djangoapps/github_sync/tests/__init__.py
index 581ac3cb25..e2b9a909a7 100644
--- a/cms/djangoapps/github_sync/tests/__init__.py
+++ b/cms/djangoapps/github_sync/tests/__init__.py
@@ -61,7 +61,7 @@ class GithubSyncTestCase(TestCase):
         self.assertIn(
             Location('i4x://edX/toy/chapter/Overview'),
             [child.location for child in self.import_course.get_children()])
-        self.assertEquals(1, len(self.import_course.get_children()))
+        self.assertEquals(2, len(self.import_course.get_children()))
 
     @patch('github_sync.sync_with_github')
     def test_sync_all_with_github(self, sync_with_github):
diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py
index 6c424c26f2..3f834c335c 100644
--- a/common/lib/xmodule/xmodule/html_module.py
+++ b/common/lib/xmodule/xmodule/html_module.py
@@ -4,9 +4,10 @@ import logging
 import os
 import sys
 from lxml import etree
+from path import path
 
 from .x_module import XModule
-from .xml_module import XmlDescriptor
+from .xml_module import XmlDescriptor, name_to_pathname
 from .editing_module import EditingDescriptor
 from .stringify import stringify_children
 from .html_checker import check_html
@@ -75,9 +76,19 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
             cls.clean_metadata_from_xml(definition_xml)
             return {'data': stringify_children(definition_xml)}
         else:
-            # html is special.  cls.filename_extension is 'xml', but if 'filename' is in the definition,
-            # that means to load from .html
-            filepath = "{category}/{name}.html".format(category='html', name=filename)
+            # html is special.  cls.filename_extension is 'xml', but
+            # if 'filename' is in the definition, that means to load
+            # from .html
+            # 'filename' in html pointers is a relative path
+            # (not same as 'html/blah.html' when the pointer is in a directory itself)
+            pointer_path = "{category}/{url_path}".format(category='html',
+                                                  url_path=name_to_pathname(location.name))
+            base = path(pointer_path).dirname()
+            #log.debug("base = {0}, base.dirname={1}, filename={2}".format(base, base.dirname(), filename))
+            filepath = "{base}/{name}.html".format(base=base, name=filename)
+            #log.debug("looking for html file for {0} at {1}".format(location, filepath))
+
+
 
             # VS[compat]
             # TODO (cpennington): If the file doesn't exist at the right path,
@@ -128,13 +139,18 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
             pass
 
         # Not proper format.  Write html to file, return an empty tag
-        filepath = u'{category}/{name}.html'.format(category=self.category,
-                                                    name=self.url_name)
+        pathname = name_to_pathname(self.url_name)
+        pathdir = path(pathname).dirname()
+        filepath = u'{category}/{pathname}.html'.format(category=self.category,
+                                                    pathname=pathname)
 
         resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
         with resource_fs.open(filepath, 'w') as file:
             file.write(self.definition['data'])
 
+        # write out the relative name
+        relname = path(pathname).basename()
+
         elt = etree.Element('html')
-        elt.set("filename", self.url_name)
+        elt.set("filename", relname)
         return elt
diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py
index d6dd85deea..c3bbc1e508 100644
--- a/common/lib/xmodule/xmodule/modulestore/__init__.py
+++ b/common/lib/xmodule/xmodule/modulestore/__init__.py
@@ -13,18 +13,21 @@ from xmodule.errortracker import ErrorLog, make_error_tracker
 
 log = logging.getLogger('mitx.' + 'modulestore')
 
+
 URL_RE = re.compile("""
     (?P<tag>[^:]+)://
     (?P<org>[^/]+)/
     (?P<course>[^/]+)/
     (?P<category>[^/]+)/
-    (?P<name>[^/]+)
-    (/(?P<revision>[^/]+))?
+    (?P<name>[^@]+)
+    (@(?P<revision>[^/]+))?
     """, re.VERBOSE)
 
 # TODO (cpennington): We should decide whether we want to expand the
 # list of valid characters in a location
 INVALID_CHARS = re.compile(r"[^\w.-]")
+# Names are allowed to have colons.
+INVALID_CHARS_NAME = re.compile(r"[^\w.:-]")
 
 _LocationBase = namedtuple('LocationBase', 'tag org course category name revision')
 
@@ -34,7 +37,7 @@ class Location(_LocationBase):
     Encodes a location.
 
     Locations representations of URLs of the
-    form {tag}://{org}/{course}/{category}/{name}[/{revision}]
+    form {tag}://{org}/{course}/{category}/{name}[@{revision}]
 
     However, they can also be represented a dictionaries (specifying each component),
     tuples or list (specified in order), or as strings of the url
@@ -81,7 +84,7 @@ class Location(_LocationBase):
 
         location - Can be any of the following types:
             string: should be of the form
-                    {tag}://{org}/{course}/{category}/{name}[/{revision}]
+                    {tag}://{org}/{course}/{category}/{name}[@{revision}]
 
             list: should be of the form [tag, org, course, category, name, revision]
 
@@ -99,10 +102,11 @@ class Location(_LocationBase):
         ommitted.
 
         Components must be composed of alphanumeric characters, or the
-        characters '_', '-', and '.'
+        characters '_', '-', and '.'.  The name component is additionally allowed to have ':',
+        which is interpreted specially for xml storage.
 
-        Components may be set to None, which may be interpreted by some contexts
-        to mean wildcard selection
+        Components may be set to None, which may be interpreted in some contexts
+        to mean wildcard selection.
         """
 
 
@@ -116,14 +120,23 @@ class Location(_LocationBase):
             return _LocationBase.__new__(_cls, *([None] * 6))
 
         def check_dict(dict_):
-            check_list(dict_.itervalues())
+            # Order matters, so flatten out into a list
+            keys = ['tag', 'org', 'course', 'category', 'name', 'revision']
+            list_ = [dict_[k] for k in keys]
+            check_list(list_)
 
         def check_list(list_):
-            for val in list_:
-                if val is not None and INVALID_CHARS.search(val) is not None:
+            def check(val, regexp):
+                if val is not None and regexp.search(val) is not None:
                     log.debug('invalid characters val="%s", list_="%s"' % (val, list_))
                     raise InvalidLocationError(location)
 
+            list_ = list(list_)
+            for val in list_[:4] + [list_[5]]:
+                check(val, INVALID_CHARS)
+            # names allow colons
+            check(list_[4], INVALID_CHARS_NAME)
+
         if isinstance(location, basestring):
             match = URL_RE.match(location)
             if match is None:
@@ -162,7 +175,7 @@ class Location(_LocationBase):
         """
         url = "{tag}://{org}/{course}/{category}/{name}".format(**self.dict())
         if self.revision:
-            url += "/" + self.revision
+            url += "@" + self.revision
         return url
 
     def html_id(self):
@@ -170,6 +183,7 @@ class Location(_LocationBase):
         Return a string with a version of the location that is safe for use in
         html id attributes
         """
+        # TODO: is ':' ok in html ids?
         return "-".join(str(v) for v in self.list()
                         if v is not None).replace('.', '_')
 
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_location.py b/common/lib/xmodule/xmodule/modulestore/tests/test_location.py
index 70c6351685..529b1f88eb 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_location.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_location.py
@@ -10,7 +10,7 @@ def check_string_roundtrip(url):
 
 def test_string_roundtrip():
     check_string_roundtrip("tag://org/course/category/name")
-    check_string_roundtrip("tag://org/course/category/name/revision")
+    check_string_roundtrip("tag://org/course/category/name@revision")
 
 
 input_dict = {
@@ -21,18 +21,28 @@ input_dict = {
     'org': 'org'
 }
 
+
+also_valid_dict = {
+    'tag': 'tag',
+    'course': 'course',
+    'category': 'category',
+    'name': 'name:more_name',
+    'org': 'org'
+}
+
+
 input_list = ['tag', 'org', 'course', 'category', 'name']
 
 input_str = "tag://org/course/category/name"
-input_str_rev = "tag://org/course/category/name/revision"
+input_str_rev = "tag://org/course/category/name@revision"
 
-valid = (input_list, input_dict, input_str, input_str_rev)
+valid = (input_list, input_dict, input_str, input_str_rev, also_valid_dict)
 
 invalid_dict = {
     'tag': 'tag',
     'course': 'course',
     'category': 'category',
-    'name': 'name/more_name',
+    'name': 'name@more_name',
     'org': 'org'
 }
 
@@ -45,8 +55,9 @@ invalid_dict2 = {
 }
 
 invalid = ("foo", ["foo"], ["foo", "bar"],
-           ["foo", "bar", "baz", "blat", "foo/bar"],
-           "tag://org/course/category/name with spaces/revision",
+           ["foo", "bar", "baz", "blat:blat", "foo:bar"],  # ':' ok in name, not in category
+           "tag://org/course/category/name with spaces@revision",
+           "tag://org/course/category/name/with/slashes@revision",
            invalid_dict,
            invalid_dict2)
 
@@ -62,16 +73,15 @@ def test_dict():
     assert_equals(dict(revision=None, **input_dict), Location(input_dict).dict())
 
     input_dict['revision'] = 'revision'
-    assert_equals("tag://org/course/category/name/revision", Location(input_dict).url())
+    assert_equals("tag://org/course/category/name@revision", Location(input_dict).url())
     assert_equals(input_dict, Location(input_dict).dict())
 
-
 def test_list():
     assert_equals("tag://org/course/category/name", Location(input_list).url())
     assert_equals(input_list + [None], Location(input_list).list())
 
     input_list.append('revision')
-    assert_equals("tag://org/course/category/name/revision", Location(input_list).url())
+    assert_equals("tag://org/course/category/name@revision", Location(input_list).url())
     assert_equals(input_list, Location(input_list).list())
 
 
@@ -87,8 +97,10 @@ def test_none():
 def test_invalid_locations():
     assert_raises(InvalidLocationError, Location, "foo")
     assert_raises(InvalidLocationError, Location, ["foo", "bar"])
+    assert_raises(InvalidLocationError, Location, ["foo", "bar", "baz", "blat/blat", "foo"])
     assert_raises(InvalidLocationError, Location, ["foo", "bar", "baz", "blat", "foo/bar"])
-    assert_raises(InvalidLocationError, Location, "tag://org/course/category/name with spaces/revision")
+    assert_raises(InvalidLocationError, Location, "tag://org/course/category/name with spaces@revision")
+    assert_raises(InvalidLocationError, Location, "tag://org/course/category/name/revision")
 
 
 def test_equality():
diff --git a/common/lib/xmodule/xmodule/tests/test_import.py b/common/lib/xmodule/xmodule/tests/test_import.py
index 3454366c1a..a369850209 100644
--- a/common/lib/xmodule/xmodule/tests/test_import.py
+++ b/common/lib/xmodule/xmodule/tests/test_import.py
@@ -255,3 +255,37 @@ class ImportTestCase(unittest.TestCase):
         two_toy_video =  modulestore.get_instance(two_toy_id, location)
         self.assertEqual(toy_video.metadata['youtube'], "1.0:p2Q6BrNhdh8")
         self.assertEqual(two_toy_video.metadata['youtube'], "1.0:p2Q6BrNhdh9")
+
+
+    def test_colon_in_url_name(self):
+        """Ensure that colons in url_names convert to file paths properly"""
+
+        print "Starting import"
+        modulestore = XMLModuleStore(DATA_DIR, eager=True, course_dirs=['toy'])
+
+        courses = modulestore.get_courses()
+        self.assertEquals(len(courses), 1)
+        course = courses[0]
+        course_id = course.id
+
+        print "course errors:"
+        for (msg, err) in modulestore.get_item_errors(course.location):
+            print msg
+            print err
+
+        chapters = course.get_children()
+        self.assertEquals(len(chapters), 2)
+
+        ch2 = chapters[1]
+        self.assertEquals(ch2.url_name, "secret:magic")
+
+        print "Ch2 location: ", ch2.location
+
+        also_ch2 = modulestore.get_instance(course_id, ch2.location)
+        self.assertEquals(ch2, also_ch2)
+
+        print "making sure html loaded"
+        cloc = course.location
+        loc = Location(cloc.tag, cloc.org, cloc.course, 'html', 'secret:toylab')
+        html = modulestore.get_instance(course_id, loc)
+        self.assertEquals(html.display_name, "Toy lab")
diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py
index b6f791ffc6..25dc4e0c6e 100644
--- a/common/lib/xmodule/xmodule/xml_module.py
+++ b/common/lib/xmodule/xmodule/xml_module.py
@@ -12,6 +12,12 @@ import sys
 
 log = logging.getLogger(__name__)
 
+def name_to_pathname(name):
+    """
+    Convert a location name for use in a path: replace ':' with '/'.
+    This allows users of the xml format to organize content into directories
+    """
+    return name.replace(':', '/')
 
 def is_pointer_tag(xml_obj):
     """
@@ -245,8 +251,8 @@ class XmlDescriptor(XModuleDescriptor):
         # VS[compat] -- detect new-style each-in-a-file mode
         if is_pointer_tag(xml_object):
             # new style:
-            # read the actual definition file--named using url_name
-            filepath = cls._format_filepath(xml_object.tag, url_name)
+            # read the actual definition file--named using url_name.replace(':','/')
+            filepath = cls._format_filepath(xml_object.tag, name_to_pathname(url_name))
             definition_xml = cls.load_file(filepath, system.resources_fs, location)
         else:
             definition_xml = xml_object	# this is just a pointer, not the real definition content
@@ -292,7 +298,8 @@ class XmlDescriptor(XModuleDescriptor):
         """If this returns True, write the definition of this descriptor to a separate
         file.
 
-        NOTE: Do not override this without a good reason.  It is here specifically for customtag...
+        NOTE: Do not override this without a good reason.  It is here
+        specifically for customtag...
         """
         return True
 
@@ -335,7 +342,8 @@ class XmlDescriptor(XModuleDescriptor):
 
         if self.export_to_file():
             # Write the definition to a file
-            filepath = self.__class__._format_filepath(self.category, self.url_name)
+            url_path = name_to_pathname(self.url_name)
+            filepath = self.__class__._format_filepath(self.category, url_path)
             resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
             with resource_fs.open(filepath, 'w') as file:
                 file.write(etree.tostring(xml_object, pretty_print=True))
diff --git a/common/test/data/toy/chapter/secret/magic.xml b/common/test/data/toy/chapter/secret/magic.xml
new file mode 100644
index 0000000000..dd16380a70
--- /dev/null
+++ b/common/test/data/toy/chapter/secret/magic.xml
@@ -0,0 +1,3 @@
+<chapter>
+    <video url_name="toyvideo" youtube="blahblah"/>
+</chapter>
diff --git a/common/test/data/toy/course/2012_Fall.xml b/common/test/data/toy/course/2012_Fall.xml
index d34eb9d56a..c87fccd9ab 100644
--- a/common/test/data/toy/course/2012_Fall.xml
+++ b/common/test/data/toy/course/2012_Fall.xml
@@ -1,9 +1,10 @@
 <course>
   <chapter url_name="Overview">
     <videosequence url_name="Toy_Videos">
-      <html url_name="toylab"/>
+      <html url_name="secret:toylab"/>
       <video url_name="Video_Resources" youtube="1.0:1bK-WdDi6Qw"/>
     </videosequence>
     <video url_name="Welcome" youtube="1.0:p2Q6BrNhdh8"/>
   </chapter>
+  <chapter url_name="secret:magic"/>
 </course>
diff --git a/common/test/data/toy/html/toylab.html b/common/test/data/toy/html/secret/toylab.html
similarity index 100%
rename from common/test/data/toy/html/toylab.html
rename to common/test/data/toy/html/secret/toylab.html
diff --git a/common/test/data/toy/html/toylab.xml b/common/test/data/toy/html/secret/toylab.xml
similarity index 100%
rename from common/test/data/toy/html/toylab.xml
rename to common/test/data/toy/html/secret/toylab.xml
diff --git a/common/test/data/toy/policies/2012_Fall.json b/common/test/data/toy/policies/2012_Fall.json
index 6c501d66f8..5ea437a8e4 100644
--- a/common/test/data/toy/policies/2012_Fall.json
+++ b/common/test/data/toy/policies/2012_Fall.json
@@ -11,7 +11,7 @@
         "display_name": "Toy Videos",
         "format": "Lecture Sequence"
     },
-    "html/toylab": {
+    "html/secret:toylab": {
         "display_name": "Toy lab"
     },
     "video/Video_Resources": {

From 84732d03b6d4856f40297a42610a4a22ea63d444 Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Mon, 27 Aug 2012 11:43:47 -0400
Subject: [PATCH 025/181] add note about ":" in format docs

---
 doc/xml-format.md | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/doc/xml-format.md b/doc/xml-format.md
index 55bcda4480..256d4839bf 100644
--- a/doc/xml-format.md
+++ b/doc/xml-format.md
@@ -189,7 +189,13 @@ This video has been encoded at 4 different speeds: 0.75x, 1x, 1.25x, and 1.5x.
 
 ## More on `url_name`s
 
-Every content element (within a course) should have a unique id.  This id is formed as `{category}/{url_name}`, or automatically generated from the content if `url_name` is not specified.  Categories are the different tag types ('chapter', 'problem', 'html', 'sequential', etc).  Url_name is a string containing a-z, A-Z, dot (.) and _.  This is what appears in urls that point to this object.
+Every content element (within a course) should have a unique id.  This id is formed as `{category}/{url_name}`, or automatically generated from the content if `url_name` is not specified.  Categories are the different tag types ('chapter', 'problem', 'html', 'sequential', etc).  Url_name is a string containing a-z, A-Z, dot (.), underscore (_), and ':'.  This is what appears in urls that point to this object.
+
+Colon (':') is special--when looking for the content definition in an xml, ':' will be replaced with '/'.  This allows organizing content into folders.  For example, given the pointer tag
+
+    <problem url_name="conceptual:add_apples_and_oranges"/>
+
+we would look for the problem definition in `problem/conceptual/add_apples_and_oranges.xml`.   (There is a technical reason why we can't just allow '/' in the url_name directly.)
 
 __IMPORTANT__: A student's state for a particular content element is tied to the element id, so the automatic id generation if only ok for elements that do not need to store any student state (e.g. verticals or customtags).  For problems, sequentials, and videos, and any other element where we keep track of what the student has done and where they are at, you should specify a unique `url_name`.  Of course, any content element that is split out into a file will need a `url_name` to specify where to find the definition.  When the CMS comes online, it will use these ids to enable content reuse, so if there is a logical name for something, please do specify it.
 

From 7b3098edc0eb8de1856c51c4a72e105b22fb9ab0 Mon Sep 17 00:00:00 2001
From: Tom Giannattasio <tom@mitx.mit.edu>
Date: Mon, 27 Aug 2012 11:48:54 -0400
Subject: [PATCH 026/181] fixed cheatsheet bug on new article page

---
 lms/templates/wiki/base.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lms/templates/wiki/base.html b/lms/templates/wiki/base.html
index 5c7381ac05..1c1bbc4a63 100644
--- a/lms/templates/wiki/base.html
+++ b/lms/templates/wiki/base.html
@@ -43,7 +43,7 @@
 	    $(document).ready(function () {
           editor = $('#div_id_content div.controls');
           cs = editor.prepend('<p class="cheatsheet">Markdown syntax is allowed. See the <a id="cheatsheetLink" href="#">cheatsheet</a> for help.</p>');
-          cs.click(function() {
+          cs.find('#cheatsheetLink').click(function() {
               $('#cheatsheetModal').modal('show');
           });
           $('#cheatsheetModal .close-btn').click(function(e) {

From b50f9759e1d147a799ce967f60f192092830ff5b Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 14:08:57 -0400
Subject: [PATCH 027/181] Catch queuekey --> queuestate KeyError in conversion

---
 common/lib/xmodule/xmodule/capa_module.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py
index 33a2f6b4c4..f13ab41c53 100644
--- a/common/lib/xmodule/xmodule/capa_module.py
+++ b/common/lib/xmodule/xmodule/capa_module.py
@@ -464,7 +464,13 @@ class CapaModule(XModule):
             raise NotFoundError('Problem must be reset before it can be checked again')
 
         # Problem queued. Students must wait XQUEUE_WAITTIME_BETWEEN_REQUESTS
-        if self.lcp.is_queued():
+        try:
+            is_queued = self.lcp.is_queued()
+        except KeyError:
+            log.info("Caught KeyError arising from 'queuekey'-->'queuestate' conversion for CapaModule name=%s" % self.name)
+            is_queued = False
+
+        if is_queued:
             current_time = datetime.datetime.now()
             prev_submit_time = self.lcp.get_recentmost_queuetime() 
             waittime_between_requests = settings.XQUEUE_WAITTIME_BETWEEN_REQUESTS

From ff38f5b390d26140f6b453450cc3d0b94480e22a Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 14:50:03 -0400
Subject: [PATCH 028/181] Debugging correctmap state

---
 common/lib/capa/capa/capa_problem.py      | 3 +++
 common/lib/xmodule/xmodule/capa_module.py | 8 +-------
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py
index ee24ebd031..3d9ee27ca0 100644
--- a/common/lib/capa/capa/capa_problem.py
+++ b/common/lib/capa/capa/capa_problem.py
@@ -14,6 +14,7 @@ This is used by capa_module.
 
 from __future__ import division
 
+import json
 import logging
 import math
 import numpy
@@ -97,6 +98,8 @@ class LoncapaProblem(object):
             if 'student_answers' in state:
                 self.student_answers = state['student_answers']
             if 'correct_map' in state:
+                print 'THK: LoncapaProblem.__init__'
+                print json.dumps(state['correct_map'], indent=4)
                 self.correct_map.set_dict(state['correct_map'])
             if 'done' in state:
                 self.done = state['done']
diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py
index f13ab41c53..33a2f6b4c4 100644
--- a/common/lib/xmodule/xmodule/capa_module.py
+++ b/common/lib/xmodule/xmodule/capa_module.py
@@ -464,13 +464,7 @@ class CapaModule(XModule):
             raise NotFoundError('Problem must be reset before it can be checked again')
 
         # Problem queued. Students must wait XQUEUE_WAITTIME_BETWEEN_REQUESTS
-        try:
-            is_queued = self.lcp.is_queued()
-        except KeyError:
-            log.info("Caught KeyError arising from 'queuekey'-->'queuestate' conversion for CapaModule name=%s" % self.name)
-            is_queued = False
-
-        if is_queued:
+        if self.lcp.is_queued():
             current_time = datetime.datetime.now()
             prev_submit_time = self.lcp.get_recentmost_queuetime() 
             waittime_between_requests = settings.XQUEUE_WAITTIME_BETWEEN_REQUESTS

From b53327a6a45d46aacf9c56714cf66d3a2c277c4e Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 15:14:12 -0400
Subject: [PATCH 029/181] Correctmap state debugging continued...

---
 common/lib/capa/capa/capa_problem.py | 1 +
 common/lib/capa/capa/correctmap.py   | 7 ++++---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py
index 3d9ee27ca0..8edc65b9f9 100644
--- a/common/lib/capa/capa/capa_problem.py
+++ b/common/lib/capa/capa/capa_problem.py
@@ -101,6 +101,7 @@ class LoncapaProblem(object):
                 print 'THK: LoncapaProblem.__init__'
                 print json.dumps(state['correct_map'], indent=4)
                 self.correct_map.set_dict(state['correct_map'])
+                print json.dumps(self.correct_map.get_dict(), indent=4)
             if 'done' in state:
                 self.done = state['done']
 
diff --git a/common/lib/capa/capa/correctmap.py b/common/lib/capa/capa/correctmap.py
index 4659e2d1e2..ed66643803 100644
--- a/common/lib/capa/capa/correctmap.py
+++ b/common/lib/capa/capa/correctmap.py
@@ -58,10 +58,11 @@ class CorrectMap(object):
         dict of dicts format.
         '''
         if correct_map and not (type(correct_map[correct_map.keys()[0]]) == dict):
-            self.__init__()							# empty current dict
-            for k in correct_map: self.set(k, correct_map[k])			# create new dict entries
+            self.__init__()	                                  # empty current dict
+            for k in correct_map: self.set(k, correct_map[k]) # create new dict entries
         else:
-            self.cmap = correct_map
+            self.__init__()
+            for k in correct_map: self.set(k, **correct_map[k])
 
     def is_correct(self, answer_id):
         if answer_id in self.cmap: return self.cmap[answer_id]['correctness'] == 'correct'

From 773a32e39f825953acea10c36deecd6177128f40 Mon Sep 17 00:00:00 2001
From: Ibrahim Awwal <ibrahim.awwal@gmail.com>
Date: Mon, 27 Aug 2012 12:59:07 -0700
Subject: [PATCH 030/181] Fix bug with sorting discussion forums (event was not
 being passed into the event handler).

---
 lms/static/coffee/src/discussion/discussion.coffee | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lms/static/coffee/src/discussion/discussion.coffee b/lms/static/coffee/src/discussion/discussion.coffee
index d8df704d1c..847f5fb97f 100644
--- a/lms/static/coffee/src/discussion/discussion.coffee
+++ b/lms/static/coffee/src/discussion/discussion.coffee
@@ -169,7 +169,7 @@ if Backbone?
       url = URI($elem.attr("action")).addSearch({text: @$(".search-input").val()})
       @reload($elem, url)
 
-    sort: ->
+    sort: (event) ->
       $elem = $(event.target)
       url = $elem.attr("sort-url")
       @reload($elem, url)

From 1e72e1c91ec68e4d689895cb12a86732e8356bc2 Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Mon, 27 Aug 2012 16:24:43 -0400
Subject: [PATCH 031/181] change 404 message

---
 lms/djangoapps/courseware/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py
index b532688037..92f6716320 100644
--- a/lms/djangoapps/courseware/views.py
+++ b/lms/djangoapps/courseware/views.py
@@ -335,7 +335,7 @@ def progress(request, course_id, student_id=None):
 
     # The course_module should be accessible, but check anyway just in case something went wrong:
     if course_module is None:
-        raise Http404("No access to this course")
+        raise Http404("Course does not exist")
 
     courseware_summary = grades.progress_summary(student, course_module,
                                                  course.grader, student_module_cache)

From 01d0697165ddeb2b3032239077c20be56497bd66 Mon Sep 17 00:00:00 2001
From: Calen Pennington <calen.pennington@gmail.com>
Date: Mon, 27 Aug 2012 16:26:35 -0400
Subject: [PATCH 032/181] Eagerly load modulestores on server startup

---
 common/lib/xmodule/xmodule/modulestore/django.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/common/lib/xmodule/xmodule/modulestore/django.py b/common/lib/xmodule/xmodule/modulestore/django.py
index 6a7315a074..59668b4a5a 100644
--- a/common/lib/xmodule/xmodule/modulestore/django.py
+++ b/common/lib/xmodule/xmodule/modulestore/django.py
@@ -43,3 +43,7 @@ def modulestore(name='default'):
         )
 
     return _MODULESTORES[name]
+
+# Initialize the modulestores immediately
+for store_name in settings.MODULESTORE:
+    modulestore(store_name)

From b45fb750b2feaff0a214228f1fa4a43a866ba817 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 17:12:20 -0400
Subject: [PATCH 033/181] CorrectMap allows unmatched keywords

---
 common/lib/capa/capa/correctmap.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/lib/capa/capa/correctmap.py b/common/lib/capa/capa/correctmap.py
index ed66643803..71f97e1321 100644
--- a/common/lib/capa/capa/correctmap.py
+++ b/common/lib/capa/capa/correctmap.py
@@ -32,7 +32,7 @@ class CorrectMap(object):
     def __iter__(self):
         return self.cmap.__iter__()
 
-    def set(self, answer_id=None, correctness=None, npoints=None, msg='', hint='', hintmode=None, queuestate=None):
+    def set(self, answer_id=None, correctness=None, npoints=None, msg='', hint='', hintmode=None, queuestate=None, **kwargs):
         if answer_id is not None:
             self.cmap[answer_id] = {'correctness': correctness,
                                     'npoints': npoints,

From d5240626528547e112c78af633c1f4494a5c6d91 Mon Sep 17 00:00:00 2001
From: David Ormsbee <dave@edx.org>
Date: Mon, 27 Aug 2012 17:47:37 -0400
Subject: [PATCH 034/181] Put quick check so we don't load course modules on
 init unless we're actually running in Django

---
 common/lib/xmodule/xmodule/modulestore/django.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/django.py b/common/lib/xmodule/xmodule/modulestore/django.py
index 59668b4a5a..d46422cbf1 100644
--- a/common/lib/xmodule/xmodule/modulestore/django.py
+++ b/common/lib/xmodule/xmodule/modulestore/django.py
@@ -5,8 +5,8 @@ Passes settings.MODULESTORE as kwargs to MongoModuleStore
 """
 
 from __future__ import absolute_import
-
 from importlib import import_module
+from os import environ
 
 from django.conf import settings
 
@@ -44,6 +44,7 @@ def modulestore(name='default'):
 
     return _MODULESTORES[name]
 
-# Initialize the modulestores immediately
-for store_name in settings.MODULESTORE:
-    modulestore(store_name)
+if 'DJANGO_SETTINGS_MODULE' in environ:
+    # Initialize the modulestores immediately
+    for store_name in settings.MODULESTORE:
+        modulestore(store_name)

From 8b0d1f09e0227fe4c4e0beed0194ae496976dc67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Andr=C3=A9s=20Rocha?= <rocha@edx.org>
Date: Mon, 27 Aug 2012 18:30:38 -0400
Subject: [PATCH 035/181] Refactored wiki cheatsheet code.

---
 lms/djangoapps/course_wiki/editors.py         | 14 ++++++++++++--
 lms/static/js/wiki/cheatsheet.js              |  9 +++++++++
 lms/static/sass/course/wiki/_wiki.scss        | 19 ++++++++++++-------
 lms/templates/wiki/base.html                  | 14 --------------
 lms/templates/wiki/create.html                |  1 +
 lms/templates/wiki/edit.html                  |  3 +++
 lms/templates/wiki/includes/cheatsheet.html   |  4 ++--
 .../wiki/includes/editor_widget.html          |  4 ++++
 8 files changed, 43 insertions(+), 25 deletions(-)
 create mode 100644 lms/static/js/wiki/cheatsheet.js
 create mode 100644 lms/templates/wiki/includes/editor_widget.html

diff --git a/lms/djangoapps/course_wiki/editors.py b/lms/djangoapps/course_wiki/editors.py
index a6583a8d98..c1f5b733ad 100644
--- a/lms/djangoapps/course_wiki/editors.py
+++ b/lms/djangoapps/course_wiki/editors.py
@@ -4,9 +4,12 @@ from django.utils.encoding import force_unicode
 from django.utils.html import conditional_escape
 from django.utils.safestring import mark_safe
 
+from django.template.loader import render_to_string
+
 from wiki.editors.base import BaseEditor
 from wiki.editors.markitup import MarkItUpAdminWidget
 
+
 class CodeMirrorWidget(forms.Widget):
     def __init__(self, attrs=None):
         # The 'rows' and 'cols' attributes are required for HTML correctness.
@@ -18,9 +21,15 @@ class CodeMirrorWidget(forms.Widget):
     
     def render(self, name, value, attrs=None):
         if value is None: value = ''
+
         final_attrs = self.build_attrs(attrs, name=name)
-        return mark_safe(u'<div><textarea%s>%s</textarea></div>' % (flatatt(final_attrs),
-                conditional_escape(force_unicode(value))))
+
+        # TODO use the help_text field of edit form instead of rendering a template
+
+        return render_to_string('wiki/includes/editor_widget.html',
+                                {'attrs': mark_safe(flatatt(final_attrs)),
+                                 'content': conditional_escape(force_unicode(value)),
+                                 })
 
 
 class CodeMirror(BaseEditor):
@@ -50,5 +59,6 @@ class CodeMirror(BaseEditor):
               "js/vendor/CodeMirror/xml.js",
               "js/vendor/CodeMirror/mitx_markdown.js",
               "js/wiki/CodeMirror.init.js",
+              "js/wiki/cheatsheet.js",
               )
 
diff --git a/lms/static/js/wiki/cheatsheet.js b/lms/static/js/wiki/cheatsheet.js
new file mode 100644
index 0000000000..e6c8e3cdaf
--- /dev/null
+++ b/lms/static/js/wiki/cheatsheet.js
@@ -0,0 +1,9 @@
+$(document).ready(function () {
+    $('#cheatsheetLink').click(function() {
+        $('#cheatsheetModal').modal('show');
+    });
+    
+    $('#cheatsheetModal .close-btn').click(function(e) {
+        $('#cheatsheetModal').modal('hide');
+    });
+});
\ No newline at end of file
diff --git a/lms/static/sass/course/wiki/_wiki.scss b/lms/static/sass/course/wiki/_wiki.scss
index a58b8fa06c..1bc38abd9a 100644
--- a/lms/static/sass/course/wiki/_wiki.scss
+++ b/lms/static/sass/course/wiki/_wiki.scss
@@ -391,6 +391,18 @@ section.wiki {
     line-height: 1.4em;
   }
 
+  #div_id_content {
+    position: relative;
+  }
+
+  #hint_id_content {
+    position: absolute;
+    top: 10px;
+    right: 0%;
+    font-size: 12px;
+    text-align:right;
+  }
+
   .CodeMirror {
     background: #fafafa;
     border: 1px solid #c8c8c8;
@@ -567,13 +579,6 @@ section.wiki {
     background: #f00 !important;
   }
 
-  .cheatsheet {
-    float: right;
-    position: relative;
-    top: -26px;
-    font-size: 12px;
-  }
-
   #cheatsheetLink {
     text-align: right;
     display: float;
diff --git a/lms/templates/wiki/base.html b/lms/templates/wiki/base.html
index 1c1bbc4a63..8a9686f546 100644
--- a/lms/templates/wiki/base.html
+++ b/lms/templates/wiki/base.html
@@ -39,18 +39,6 @@
     {% with mathjax_mode='wiki' %}
       {% include "mathjax_include.html" %}
     {% endwith %}
-    <script type="text/javascript">
-	    $(document).ready(function () {
-          editor = $('#div_id_content div.controls');
-          cs = editor.prepend('<p class="cheatsheet">Markdown syntax is allowed. See the <a id="cheatsheetLink" href="#">cheatsheet</a> for help.</p>');
-          cs.find('#cheatsheetLink').click(function() {
-              $('#cheatsheetModal').modal('show');
-          });
-          $('#cheatsheetModal .close-btn').click(function(e) {
-            $('#cheatsheetModal').modal('hide');
-          });
-	    });
-    </script>
   {% endaddtoblock %}
 
 {% endblock %}
@@ -81,8 +69,6 @@
       {% endblock %}
     </div>
 
-    {% include "wiki/includes/cheatsheet.html" %}
-
   </section>
 
 {% endblock %}
diff --git a/lms/templates/wiki/create.html b/lms/templates/wiki/create.html
index 886764ba84..745be08cf8 100644
--- a/lms/templates/wiki/create.html
+++ b/lms/templates/wiki/create.html
@@ -42,6 +42,7 @@
             {% trans "Go back" %}
           </a>
       </div>  
+      {% include "wiki/includes/cheatsheet.html" %}
     </form>
 
   </article>
diff --git a/lms/templates/wiki/edit.html b/lms/templates/wiki/edit.html
index f4bd7d138f..65378da4e4 100644
--- a/lms/templates/wiki/edit.html
+++ b/lms/templates/wiki/edit.html
@@ -40,7 +40,10 @@
       </a>
     </div>
   </div>
+  {% include "wiki/includes/cheatsheet.html" %}
 </form>
 
 {% endblock %}
 
+
+
diff --git a/lms/templates/wiki/includes/cheatsheet.html b/lms/templates/wiki/includes/cheatsheet.html
index 429c1ea22e..6f5170e601 100644
--- a/lms/templates/wiki/includes/cheatsheet.html
+++ b/lms/templates/wiki/includes/cheatsheet.html
@@ -1,4 +1,4 @@
-<section id="cheatsheetModal" class="modal hide fade">
+<div class="modal hide fade" id="cheatsheetModal">
 	<a href="#" class="close-btn">×</a>
 	<div id="cheatsheet-body" class="modal-body">
 		<div class="left-column">
@@ -53,4 +53,4 @@ Smaller Header
 
 		</div>
 	</div>
-</section>
+</div>
diff --git a/lms/templates/wiki/includes/editor_widget.html b/lms/templates/wiki/includes/editor_widget.html
new file mode 100644
index 0000000000..2f5dd7ce68
--- /dev/null
+++ b/lms/templates/wiki/includes/editor_widget.html
@@ -0,0 +1,4 @@
+<textarea {{ attrs }}>{{ content }}</textarea>
+<p id="hint_id_content" class="help-block">
+  Markdown syntax is allowed. See the <a id="cheatsheetLink" href="#">cheatsheet</a> for help.
+</p>

From 57c6bfc3a6239e91caaf2237b98deb862127e27e Mon Sep 17 00:00:00 2001
From: David Ormsbee <dave@edx.org>
Date: Mon, 27 Aug 2012 18:46:13 -0400
Subject: [PATCH 036/181] commenting out the pre-loading of modules until we
 get a better way in place (that doesn't hang on deploy)

---
 common/lib/xmodule/xmodule/modulestore/django.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/django.py b/common/lib/xmodule/xmodule/modulestore/django.py
index d46422cbf1..0b86c2fea4 100644
--- a/common/lib/xmodule/xmodule/modulestore/django.py
+++ b/common/lib/xmodule/xmodule/modulestore/django.py
@@ -44,7 +44,7 @@ def modulestore(name='default'):
 
     return _MODULESTORES[name]
 
-if 'DJANGO_SETTINGS_MODULE' in environ:
-    # Initialize the modulestores immediately
-    for store_name in settings.MODULESTORE:
-        modulestore(store_name)
+# if 'DJANGO_SETTINGS_MODULE' in environ:
+#     # Initialize the modulestores immediately
+#     for store_name in settings.MODULESTORE:
+#         modulestore(store_name)

From c462b1a917292f400db7a2cf126622e22a04fce2 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 18:50:31 -0400
Subject: [PATCH 037/181] Document CorrectMap.set

---
 common/lib/capa/capa/correctmap.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/common/lib/capa/capa/correctmap.py b/common/lib/capa/capa/correctmap.py
index 71f97e1321..112976389c 100644
--- a/common/lib/capa/capa/correctmap.py
+++ b/common/lib/capa/capa/correctmap.py
@@ -32,6 +32,7 @@ class CorrectMap(object):
     def __iter__(self):
         return self.cmap.__iter__()
 
+    # See the documentation for 'set_dict' for the use of kwargs
     def set(self, answer_id=None, correctness=None, npoints=None, msg='', hint='', hintmode=None, queuestate=None, **kwargs):
         if answer_id is not None:
             self.cmap[answer_id] = {'correctness': correctness,
@@ -53,9 +54,19 @@ class CorrectMap(object):
 
     def set_dict(self, correct_map):
         '''
-        set internal dict to provided correct_map dict
-        for graceful migration, if correct_map is a one-level dict, then convert it to the new
-        dict of dicts format.
+        Set internal dict of CorrectMap to provided correct_map dict
+
+        correct_map is saved by LMS as a plaintext JSON dump of the correctmap dict. This means that
+        when the definition of CorrectMap (e.g. its properties) are altered, existing correct_map dict
+        not coincide with the newest CorrectMap format as defined by self.set.
+
+        For graceful migration, feed the contents of each correct map to self.set, rather than
+        making a direct copy of the given correct_map dict. This way, the common keys between 
+        the incoming correct_map dict and the new CorrectMap instance will be written, while
+        mismatched keys will be gracefully ignored.
+
+        Special migration case:
+            If correct_map is a one-level dict, then convert it to the new dict of dicts format.
         '''
         if correct_map and not (type(correct_map[correct_map.keys()[0]]) == dict):
             self.__init__()	                                  # empty current dict

From 0cff6fa30dd3ece7af80a3b73e4f200274f84532 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 18:51:38 -0400
Subject: [PATCH 038/181] Remove debugging comment

---
 common/lib/capa/capa/capa_problem.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py
index 8edc65b9f9..d7bae1a36c 100644
--- a/common/lib/capa/capa/capa_problem.py
+++ b/common/lib/capa/capa/capa_problem.py
@@ -98,10 +98,7 @@ class LoncapaProblem(object):
             if 'student_answers' in state:
                 self.student_answers = state['student_answers']
             if 'correct_map' in state:
-                print 'THK: LoncapaProblem.__init__'
-                print json.dumps(state['correct_map'], indent=4)
                 self.correct_map.set_dict(state['correct_map'])
-                print json.dumps(self.correct_map.get_dict(), indent=4)
             if 'done' in state:
                 self.done = state['done']
 

From dca16ad007b60f093c36cbf126e986e1e85730bb Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 19:01:36 -0400
Subject: [PATCH 039/181] CorrectMap uses dict to represent queuestate

---
 common/lib/capa/capa/correctmap.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/common/lib/capa/capa/correctmap.py b/common/lib/capa/capa/correctmap.py
index 112976389c..07454471f0 100644
--- a/common/lib/capa/capa/correctmap.py
+++ b/common/lib/capa/capa/correctmap.py
@@ -15,7 +15,7 @@ class CorrectMap(object):
     - msg         : string (may have HTML) giving extra message response (displayed below textline or textbox)
     - hint        : string (may have HTML) giving optional hint (displayed below textline or textbox, above msg)
     - hintmode    : one of (None,'on_request','always') criteria for displaying hint
-    - queuestate  : Tuple (key, time) where key is a secret string, and time is a string dump
+    - queuestate  : Dict {key:'', time:''} where key is a secret string, and time is a string dump
                     of a DateTime object in the format '%Y%m%d%H%M%S'. Is None when not queued
 
     Behaves as a dict.
@@ -83,10 +83,10 @@ class CorrectMap(object):
         return answer_id in self.cmap and self.cmap[answer_id]['queuestate'] is not None
 
     def is_right_queuekey(self, answer_id, test_key):
-        return self.is_queued(answer_id) and self.cmap[answer_id]['queuestate'][0] == test_key
+        return self.is_queued(answer_id) and self.cmap[answer_id]['queuestate']['key'] == test_key
 
     def get_queuetime_str(self, answer_id):
-        return self.cmap[answer_id]['queuestate'][1] if self.is_queued(answer_id) else None 
+        return self.cmap[answer_id]['queuestate']['time'] if self.is_queued(answer_id) else None 
 
     def get_npoints(self, answer_id):
         npoints = self.get_property(answer_id, 'npoints')

From 50481c2a81f6659ee162a0e12b8bd69e998a52e1 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 19:04:36 -0400
Subject: [PATCH 040/181] CodeResponse saves queuestate as dict

---
 common/lib/capa/capa/responsetypes.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index 5cf45adce2..c997cec30b 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -1151,7 +1151,9 @@ class CodeResponse(LoncapaResponse):
 
         # State associated with the queueing request
         qtime = datetime.strftime(datetime.now(), '%Y%m%d%H%M%S')
-        queuestate = (queuekey, qtime)
+        queuestate = {'key': queuekey,
+                      'time': qtime,
+                     }
 
         cmap = CorrectMap() 
         if error:

From a9a73561f434972515984eff34d7d30b7c3afddf Mon Sep 17 00:00:00 2001
From: John Hess <mgojohn@gmail.com>
Date: Mon, 27 Aug 2012 19:05:00 -0400
Subject: [PATCH 041/181] updated admin_dashboard to show count of unique
 students

---
 lms/djangoapps/dashboard/views.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/lms/djangoapps/dashboard/views.py b/lms/djangoapps/dashboard/views.py
index c4446bceaa..a1a3461953 100644
--- a/lms/djangoapps/dashboard/views.py
+++ b/lms/djangoapps/dashboard/views.py
@@ -21,11 +21,16 @@ def dashboard(request):
     if not request.user.is_staff:
         raise Http404
 
-    query = "select count(user_id) as students, course_id from student_courseenrollment group by course_id order by students desc"
+    queries=[]
+    queries.append("select count(user_id) as students, course_id from student_courseenrollment group by course_id order by students desc;")
+    queries.append("select count(distinct user_id) as unique_students from student_courseenrollment;")
 
     from django.db import connection
     cursor = connection.cursor()
-    cursor.execute(query)
-    results = dictfetchall(cursor)
+    results =[]
+
+    for query in queries:
+        cursor.execute(query)
+        results.append(dictfetchall(cursor))
 
     return HttpResponse(json.dumps(results, indent=4))

From 98542f79ece0d1755a1a04b9041f995784227dff Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 19:14:47 -0400
Subject: [PATCH 042/181] Define dateformat variable in xqueue_interface

---
 common/lib/capa/capa/capa_problem.py     | 7 +++++--
 common/lib/capa/capa/responsetypes.py    | 2 +-
 common/lib/capa/capa/xqueue_interface.py | 2 +-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py
index d7bae1a36c..9dc7658884 100644
--- a/common/lib/capa/capa/capa_problem.py
+++ b/common/lib/capa/capa/capa_problem.py
@@ -34,6 +34,7 @@ from datetime import datetime
 import eia
 import inputtypes
 from util import contextualize_text, convert_files_to_filenames
+import xqueue_interface
 
 # to be replaced with auto-registering
 import responsetypes
@@ -215,8 +216,10 @@ class LoncapaProblem(object):
             return None
 
         # Get a list of timestamps of all queueing requests, then convert it to a DateTime object
-        queuetimes = [self.correct_map.get_queuetime_str(answer_id) for answer_id in self.correct_map if self.correct_map.is_queued(answer_id)]
-        queuetimes = [datetime.strptime(qt,'%Y%m%d%H%M%S') for qt in queuetimes]
+        queuetime_strs = [self.correct_map.get_queuetime_str(answer_id)
+                          for answer_id in self.correct_map 
+                          if self.correct_map.is_queued(answer_id)]
+        queuetimes = [datetime.strptime(qt_str, xqueue_interface.dateformat) for qt_str in queuetime_strs]
 
         return max(queuetimes)
 
diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index c997cec30b..2ab5f3eafd 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -1150,7 +1150,7 @@ class CodeResponse(LoncapaResponse):
                                                     body=json.dumps(contents))
 
         # State associated with the queueing request
-        qtime = datetime.strftime(datetime.now(), '%Y%m%d%H%M%S')
+        qtime = datetime.strftime(datetime.now(), xqueue_interface.dateformat)
         queuestate = {'key': queuekey,
                       'time': qtime,
                      }
diff --git a/common/lib/capa/capa/xqueue_interface.py b/common/lib/capa/capa/xqueue_interface.py
index 2930eb682d..519f63cacd 100644
--- a/common/lib/capa/capa/xqueue_interface.py
+++ b/common/lib/capa/capa/xqueue_interface.py
@@ -9,7 +9,7 @@ import time
 
 
 log = logging.getLogger('mitx.' + __name__)
-
+dateformat = '%Y%m%d%H%M%S'
 
 def make_hashkey(seed=None):
     '''

From 4468fee8e5948d63506b2bfe6e6adb5239cc8000 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 19:15:28 -0400
Subject: [PATCH 043/181] Define dateformat variable in xqueue_interface

---
 common/lib/capa/capa/capa_problem.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py
index 9dc7658884..3a847f8306 100644
--- a/common/lib/capa/capa/capa_problem.py
+++ b/common/lib/capa/capa/capa_problem.py
@@ -205,7 +205,7 @@ class LoncapaProblem(object):
         '''
         Returns True if any part of the problem has been submitted to an external queue
         '''
-        return any([self.correct_map.is_queued(answer_id) for answer_id in self.correct_map])
+        return any(self.correct_map.is_queued(answer_id) for answer_id in self.correct_map)
 
 
     def get_recentmost_queuetime(self):

From 03588c094c355422821238cf81e3aa0a49934c6e Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 19:17:00 -0400
Subject: [PATCH 044/181] Move stdlib import location

---
 common/lib/capa/capa/capa_problem.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py
index 3a847f8306..f386c9fe24 100644
--- a/common/lib/capa/capa/capa_problem.py
+++ b/common/lib/capa/capa/capa_problem.py
@@ -14,6 +14,7 @@ This is used by capa_module.
 
 from __future__ import division
 
+from datetime import datetime
 import json
 import logging
 import math
@@ -30,7 +31,6 @@ from xml.sax.saxutils import unescape
 
 import calc
 from correctmap import CorrectMap
-from datetime import datetime
 import eia
 import inputtypes
 from util import contextualize_text, convert_files_to_filenames

From 572e88d5b20fb5a7d326623615922885c328c3af Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 19:20:50 -0400
Subject: [PATCH 045/181] Adjust wait message

---
 common/lib/xmodule/xmodule/capa_module.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py
index 33a2f6b4c4..0f97de6f3f 100644
--- a/common/lib/xmodule/xmodule/capa_module.py
+++ b/common/lib/xmodule/xmodule/capa_module.py
@@ -469,7 +469,7 @@ class CapaModule(XModule):
             prev_submit_time = self.lcp.get_recentmost_queuetime() 
             waittime_between_requests = settings.XQUEUE_WAITTIME_BETWEEN_REQUESTS
             if (current_time-prev_submit_time).total_seconds() < waittime_between_requests:
-                msg = 'You must wait at least %d seconds between queue submissions' % waittime_between_requests
+                msg = 'You must wait at least %d seconds between submissions' % waittime_between_requests
                 return {'success': msg, 'html': ''}
 
         try:

From 2cf4b28cf721b96704d0c82387ff371c56fce4c9 Mon Sep 17 00:00:00 2001
From: John Hess <mgojohn@gmail.com>
Date: Mon, 27 Aug 2012 19:22:19 -0400
Subject: [PATCH 046/181] added additional query to show number of users with
 any given number of registrations (power users)

---
 lms/djangoapps/dashboard/views.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lms/djangoapps/dashboard/views.py b/lms/djangoapps/dashboard/views.py
index a1a3461953..964b3fac4a 100644
--- a/lms/djangoapps/dashboard/views.py
+++ b/lms/djangoapps/dashboard/views.py
@@ -24,7 +24,8 @@ def dashboard(request):
     queries=[]
     queries.append("select count(user_id) as students, course_id from student_courseenrollment group by course_id order by students desc;")
     queries.append("select count(distinct user_id) as unique_students from student_courseenrollment;")
-
+    queries.append("select registrations, count(registrations) from (select count(user_id) as registrations from student_courseenrollment group by user_id) as registrations_per_user group by registrations;")
+    
     from django.db import connection
     cursor = connection.cursor()
     results =[]

From a197f634ea8047da38cd7b388de230ce948a96a4 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 19:34:46 -0400
Subject: [PATCH 047/181] Get_queuetime_str does not return None

---
 common/lib/capa/capa/correctmap.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/lib/capa/capa/correctmap.py b/common/lib/capa/capa/correctmap.py
index 07454471f0..52411a8e8c 100644
--- a/common/lib/capa/capa/correctmap.py
+++ b/common/lib/capa/capa/correctmap.py
@@ -86,7 +86,7 @@ class CorrectMap(object):
         return self.is_queued(answer_id) and self.cmap[answer_id]['queuestate']['key'] == test_key
 
     def get_queuetime_str(self, answer_id):
-        return self.cmap[answer_id]['queuestate']['time'] if self.is_queued(answer_id) else None 
+        return self.cmap[answer_id]['queuestate']['time']
 
     def get_npoints(self, answer_id):
         npoints = self.get_property(answer_id, 'npoints')

From 26051f9939c4f4597d56146c70ed743f0ede3e72 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 19:37:46 -0400
Subject: [PATCH 048/181] Waittime passed through ModuleSystem.xqueue

---
 common/lib/xmodule/xmodule/capa_module.py  | 3 +--
 lms/djangoapps/courseware/module_render.py | 4 +++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py
index 0f97de6f3f..ef3de02b37 100644
--- a/common/lib/xmodule/xmodule/capa_module.py
+++ b/common/lib/xmodule/xmodule/capa_module.py
@@ -8,7 +8,6 @@ import re
 import sys
 
 from datetime import timedelta
-from django.conf import settings
 from lxml import etree
 from pkg_resources import resource_string
 
@@ -467,7 +466,7 @@ class CapaModule(XModule):
         if self.lcp.is_queued():
             current_time = datetime.datetime.now()
             prev_submit_time = self.lcp.get_recentmost_queuetime() 
-            waittime_between_requests = settings.XQUEUE_WAITTIME_BETWEEN_REQUESTS
+            waittime_between_requests = self.system.xqueue.waittime
             if (current_time-prev_submit_time).total_seconds() < waittime_between_requests:
                 msg = 'You must wait at least %d seconds between submissions' % waittime_between_requests
                 return {'success': msg, 'html': ''}
diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py
index da9828fb12..0eaa1ee60f 100644
--- a/lms/djangoapps/courseware/module_render.py
+++ b/lms/djangoapps/courseware/module_render.py
@@ -217,7 +217,9 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
 
     xqueue = {'interface': xqueue_interface,
               'callback_url': xqueue_callback_url,
-              'default_queuename': xqueue_default_queuename.replace(' ', '_')}
+              'default_queuename': xqueue_default_queuename.replace(' ', '_'),
+              'waittime': settings.XQUEUE_WAITTIME_BETWEEN_REQUESTS
+             }
 
     def inner_get_module(location):
         """

From 3051c6aca719be45d5cf7bc977ef4624fa3a8f3d Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 19:41:35 -0400
Subject: [PATCH 049/181] Access system.xqueue properly as dict

---
 common/lib/xmodule/xmodule/capa_module.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py
index ef3de02b37..27641af3b5 100644
--- a/common/lib/xmodule/xmodule/capa_module.py
+++ b/common/lib/xmodule/xmodule/capa_module.py
@@ -466,7 +466,7 @@ class CapaModule(XModule):
         if self.lcp.is_queued():
             current_time = datetime.datetime.now()
             prev_submit_time = self.lcp.get_recentmost_queuetime() 
-            waittime_between_requests = self.system.xqueue.waittime
+            waittime_between_requests = self.system.xqueue['waittime']
             if (current_time-prev_submit_time).total_seconds() < waittime_between_requests:
                 msg = 'You must wait at least %d seconds between submissions' % waittime_between_requests
                 return {'success': msg, 'html': ''}

From 093f08064114acf398118240bc7819c15c32a13a Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 19:45:39 -0400
Subject: [PATCH 050/181] Filesubmission input does not reveal queue length

---
 common/lib/capa/capa/inputtypes.py                 | 2 +-
 common/lib/capa/capa/templates/filesubmission.html | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/common/lib/capa/capa/inputtypes.py b/common/lib/capa/capa/inputtypes.py
index 9290503c9d..a323a5dbfe 100644
--- a/common/lib/capa/capa/inputtypes.py
+++ b/common/lib/capa/capa/inputtypes.py
@@ -351,7 +351,7 @@ def filesubmission(element, value, status, render_template, msg=''):
     if status == 'incomplete': # Flag indicating that the problem has been queued, 'msg' is length of queue
         status = 'queued'
         queue_len = msg
-        msg = 'Submitted to grader. (Queue length: %s)' % queue_len
+        msg = 'Submitted to grader.'
 
     context = { 'id': eid, 'state': status, 'msg': msg, 'value': value,
             'queue_len': queue_len, 'allowed_files': allowed_files,
diff --git a/common/lib/capa/capa/templates/filesubmission.html b/common/lib/capa/capa/templates/filesubmission.html
index a859dc8458..e9fd7c5674 100644
--- a/common/lib/capa/capa/templates/filesubmission.html
+++ b/common/lib/capa/capa/templates/filesubmission.html
@@ -10,7 +10,7 @@
     <span class="processing" id="status_${id}"></span>
     <span style="display:none;" class="xqueue" id="${id}" >${queue_len}</span>
     % endif
-    <span class="debug">(${state})</span>
+    <span style="display:none;" class="debug">(${state})</span>
     <br/>
     <span class="message">${msg|n}</span>
     <br/>

From 7c5879a1a58fcf291f63be0acc8943fd76db569e Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 19:50:47 -0400
Subject: [PATCH 051/181] Use os.path.join

---
 common/lib/xmodule/xmodule/tests/__init__.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/common/lib/xmodule/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py
index bd5d35c75f..474ee1280a 100644
--- a/common/lib/xmodule/xmodule/tests/__init__.py
+++ b/common/lib/xmodule/xmodule/tests/__init__.py
@@ -19,6 +19,7 @@ import capa.calc as calc
 import capa.capa_problem as lcp
 from capa.correctmap import CorrectMap
 from capa.util import convert_files_to_filenames
+from capa.xqueue_interface import dateformat
 from datetime import datetime
 from xmodule import graders, x_module
 from xmodule.x_module import ModuleSystem
@@ -36,7 +37,7 @@ i4xs = ModuleSystem(
     user=Mock(),
     filestore=fs.osfs.OSFS(os.path.dirname(os.path.realpath(__file__))+"/test_files"),
     debug=True,
-    xqueue={'interface':None, 'callback_url':'/', 'default_queuename': 'testqueue'},
+    xqueue={'interface':None, 'callback_url':'/', 'default_queuename': 'testqueue', 'waittime': 10},
     node_path=os.environ.get("NODE_PATH", "/usr/local/lib/node_modules")
 )
 
@@ -286,14 +287,14 @@ class CodeResponseTest(unittest.TestCase):
     '''
     @staticmethod
     def make_queuestate(key, time):
-        timestr = datetime.strftime(time,'%Y%m%d%H%M%S')
-        return (key, timestr)
+        timestr = datetime.strftime(time, dateformat)
+        return {'key': key, 'time': timestr}
 
     def test_is_queued(self):
         ''' 
         Simple test of whether LoncapaProblem knows when it's been queued
         '''
-        problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
+        problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
         test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs)
         
         answer_ids = sorted(test_lcp.get_question_answers().keys())

From 2919389d8e5cabfb066960818231e0a2360949a4 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 19:53:10 -0400
Subject: [PATCH 052/181] Use os.path.join -- all of them...

---
 common/lib/xmodule/xmodule/tests/__init__.py | 41 ++++++++++----------
 1 file changed, 21 insertions(+), 20 deletions(-)

diff --git a/common/lib/xmodule/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py
index 474ee1280a..5b1843a632 100644
--- a/common/lib/xmodule/xmodule/tests/__init__.py
+++ b/common/lib/xmodule/xmodule/tests/__init__.py
@@ -295,33 +295,34 @@ class CodeResponseTest(unittest.TestCase):
         Simple test of whether LoncapaProblem knows when it's been queued
         '''
         problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
-        test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs)
-        
-        answer_ids = sorted(test_lcp.get_question_answers().keys())
-        num_answers = len(answer_ids)
+        with open(problem_file) as input_file:
+            test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
+            
+            answer_ids = sorted(test_lcp.get_question_answers().keys())
+            num_answers = len(answer_ids)
 
-        # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
-        cmap = CorrectMap()
-        for i in range(num_answers):
-            cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=None))
-        test_lcp.correct_map.update(cmap)
+            # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
+            cmap = CorrectMap()
+            for i in range(num_answers):
+                cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=None))
+            test_lcp.correct_map.update(cmap)
 
-        self.assertEquals(test_lcp.is_queued(), False)
+            self.assertEquals(test_lcp.is_queued(), False)
 
-        # Now we queue the LCP
-        cmap = CorrectMap()
-        for i in range(num_answers):
-            queuestate = CodeResponseTest.make_queuestate(i, datetime.now())
-            cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
-        test_lcp.correct_map.update(cmap)
+            # Now we queue the LCP
+            cmap = CorrectMap()
+            for i in range(num_answers):
+                queuestate = CodeResponseTest.make_queuestate(i, datetime.now())
+                cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
+            test_lcp.correct_map.update(cmap)
 
-        self.assertEquals(test_lcp.is_queued(), True)
+            self.assertEquals(test_lcp.is_queued(), True)
 
     def test_update_score(self):
         '''
         Test whether LoncapaProblem.update_score can deliver queued result to the right subproblem
         '''
-        problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
+        problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
         test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs)
 
         answer_ids = sorted(test_lcp.get_question_answers().keys())
@@ -377,7 +378,7 @@ class CodeResponseTest(unittest.TestCase):
         '''
         Test whether the LoncapaProblem knows about the time of queue requests
         '''
-        problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
+        problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
         test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs)
 
         answer_ids = sorted(test_lcp.get_question_answers().keys())
@@ -411,7 +412,7 @@ class CodeResponseTest(unittest.TestCase):
         '''
         Test whether file objects are converted to filenames without altering other structures
         '''
-        problem_file = os.path.dirname(__file__) + "/test_files/coderesponse.xml"
+        problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
         fp = open(problem_file)
         answers_with_file = {'1_2_1': 'String-based answer',
                              '1_3_1': ['answer1', 'answer2', 'answer3'],

From b4b8f6bc7d69d2771bf287cd2fc987807f821630 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 19:54:58 -0400
Subject: [PATCH 053/181] Use with open(file) structure

---
 common/lib/xmodule/xmodule/tests/__init__.py | 156 ++++++++++---------
 1 file changed, 79 insertions(+), 77 deletions(-)

diff --git a/common/lib/xmodule/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py
index 5b1843a632..5385b489b2 100644
--- a/common/lib/xmodule/xmodule/tests/__init__.py
+++ b/common/lib/xmodule/xmodule/tests/__init__.py
@@ -323,55 +323,56 @@ class CodeResponseTest(unittest.TestCase):
         Test whether LoncapaProblem.update_score can deliver queued result to the right subproblem
         '''
         problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
-        test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs)
+        with open(problem_file) as input_file:
+            test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
 
-        answer_ids = sorted(test_lcp.get_question_answers().keys())
-        num_answers = len(answer_ids)
-
-        # CodeResponse requires internal CorrectMap state. Build it now in the queued state
-        old_cmap = CorrectMap()
-        for i in range(num_answers):
-            queuekey = 1000 + i
-            queuestate = CodeResponseTest.make_queuestate(1000+i, datetime.now())
-            old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
-
-        # Message format common to external graders
-        correct_score_msg = json.dumps({'correct':True, 'score':1, 'msg':'MESSAGE'})
-        incorrect_score_msg = json.dumps({'correct':False, 'score':0, 'msg':'MESSAGE'})
-
-        xserver_msgs = {'correct': correct_score_msg,
-                        'incorrect': incorrect_score_msg,}
-
-        # Incorrect queuekey, state should not be updated
-        for correctness in ['correct', 'incorrect']:
-            test_lcp.correct_map = CorrectMap()
-            test_lcp.correct_map.update(old_cmap)  # Deep copy
-
-            test_lcp.update_score(xserver_msgs[correctness], queuekey=0)
-            self.assertEquals(test_lcp.correct_map.get_dict(), old_cmap.get_dict())  # Deep comparison
+            answer_ids = sorted(test_lcp.get_question_answers().keys())
+            num_answers = len(answer_ids)
 
+            # CodeResponse requires internal CorrectMap state. Build it now in the queued state
+            old_cmap = CorrectMap()
             for i in range(num_answers):
-                self.assertTrue(test_lcp.correct_map.is_queued(answer_ids[i]))  # Should be still queued, since message undelivered
+                queuekey = 1000 + i
+                queuestate = CodeResponseTest.make_queuestate(1000+i, datetime.now())
+                old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
 
-        # Correct queuekey, state should be updated
-        for correctness in ['correct', 'incorrect']:
-            for i in range(num_answers):  # Target specific answer_id's
+            # Message format common to external graders
+            correct_score_msg = json.dumps({'correct':True, 'score':1, 'msg':'MESSAGE'})
+            incorrect_score_msg = json.dumps({'correct':False, 'score':0, 'msg':'MESSAGE'})
+
+            xserver_msgs = {'correct': correct_score_msg,
+                            'incorrect': incorrect_score_msg,}
+
+            # Incorrect queuekey, state should not be updated
+            for correctness in ['correct', 'incorrect']:
                 test_lcp.correct_map = CorrectMap()
-                test_lcp.correct_map.update(old_cmap)
+                test_lcp.correct_map.update(old_cmap)  # Deep copy
 
-                new_cmap = CorrectMap()
-                new_cmap.update(old_cmap)
-                npoints = 1 if correctness=='correct' else 0
-                new_cmap.set(answer_id=answer_ids[i], npoints=npoints, correctness=correctness, msg='MESSAGE', queuestate=None)
+                test_lcp.update_score(xserver_msgs[correctness], queuekey=0)
+                self.assertEquals(test_lcp.correct_map.get_dict(), old_cmap.get_dict())  # Deep comparison
 
-                test_lcp.update_score(xserver_msgs[correctness], queuekey=1000 + i)
-                self.assertEquals(test_lcp.correct_map.get_dict(), new_cmap.get_dict())
+                for i in range(num_answers):
+                    self.assertTrue(test_lcp.correct_map.is_queued(answer_ids[i]))  # Should be still queued, since message undelivered
 
-                for j in range(num_answers):
-                    if j == i:
-                        self.assertFalse(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be dequeued, message delivered
-                    else:
-                        self.assertTrue(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be queued, message undelivered
+            # Correct queuekey, state should be updated
+            for correctness in ['correct', 'incorrect']:
+                for i in range(num_answers):  # Target specific answer_id's
+                    test_lcp.correct_map = CorrectMap()
+                    test_lcp.correct_map.update(old_cmap)
+
+                    new_cmap = CorrectMap()
+                    new_cmap.update(old_cmap)
+                    npoints = 1 if correctness=='correct' else 0
+                    new_cmap.set(answer_id=answer_ids[i], npoints=npoints, correctness=correctness, msg='MESSAGE', queuestate=None)
+
+                    test_lcp.update_score(xserver_msgs[correctness], queuekey=1000 + i)
+                    self.assertEquals(test_lcp.correct_map.get_dict(), new_cmap.get_dict())
+
+                    for j in range(num_answers):
+                        if j == i:
+                            self.assertFalse(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be dequeued, message delivered
+                        else:
+                            self.assertTrue(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be queued, message undelivered
     
 
     def test_recentmost_queuetime(self):
@@ -379,48 +380,49 @@ class CodeResponseTest(unittest.TestCase):
         Test whether the LoncapaProblem knows about the time of queue requests
         '''
         problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
-        test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=i4xs)
+        with open(problem_file) as input_file:
+            test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
 
-        answer_ids = sorted(test_lcp.get_question_answers().keys())
-        num_answers = len(answer_ids)
+            answer_ids = sorted(test_lcp.get_question_answers().keys())
+            num_answers = len(answer_ids)
 
-        # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
-        cmap = CorrectMap()
-        for i in range(num_answers):
-            cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=None))
-        test_lcp.correct_map.update(cmap)
-        
-        self.assertEquals(test_lcp.get_recentmost_queuetime(), None)
+            # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
+            cmap = CorrectMap()
+            for i in range(num_answers):
+                cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=None))
+            test_lcp.correct_map.update(cmap)
+            
+            self.assertEquals(test_lcp.get_recentmost_queuetime(), None)
 
-        # CodeResponse requires internal CorrectMap state. Build it now in the queued state
-        cmap = CorrectMap()
-        answer_ids = sorted(test_lcp.get_question_answers().keys())
-        num_answers = len(answer_ids)
-        for i in range(num_answers):
-            queuekey = 1000 + i
-            latest_timestamp = datetime.now()
-            queuestate = CodeResponseTest.make_queuestate(1000+i, latest_timestamp)
-            cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
-        test_lcp.correct_map.update(cmap)
+            # CodeResponse requires internal CorrectMap state. Build it now in the queued state
+            cmap = CorrectMap()
+            answer_ids = sorted(test_lcp.get_question_answers().keys())
+            num_answers = len(answer_ids)
+            for i in range(num_answers):
+                queuekey = 1000 + i
+                latest_timestamp = datetime.now()
+                queuestate = CodeResponseTest.make_queuestate(1000+i, latest_timestamp)
+                cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
+            test_lcp.correct_map.update(cmap)
 
-        # Queue state only tracks up to second
-        latest_timestamp = datetime.strptime(datetime.strftime(latest_timestamp,'%Y%m%d%H%M%S'),'%Y%m%d%H%M%S')
+            # Queue state only tracks up to second
+            latest_timestamp = datetime.strptime(datetime.strftime(latest_timestamp,'%Y%m%d%H%M%S'),'%Y%m%d%H%M%S')
 
-        self.assertEquals(test_lcp.get_recentmost_queuetime(), latest_timestamp)
+            self.assertEquals(test_lcp.get_recentmost_queuetime(), latest_timestamp)
 
-    def test_convert_files_to_filenames(self):
-        '''
-        Test whether file objects are converted to filenames without altering other structures
-        '''
-        problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
-        fp = open(problem_file)
-        answers_with_file = {'1_2_1': 'String-based answer',
-                             '1_3_1': ['answer1', 'answer2', 'answer3'],
-                             '1_4_1': [fp, fp]}
-        answers_converted = convert_files_to_filenames(answers_with_file)
-        self.assertEquals(answers_converted['1_2_1'], 'String-based answer')
-        self.assertEquals(answers_converted['1_3_1'], ['answer1', 'answer2', 'answer3'])
-        self.assertEquals(answers_converted['1_4_1'], [fp.name, fp.name])
+        def test_convert_files_to_filenames(self):
+            '''
+            Test whether file objects are converted to filenames without altering other structures
+            '''
+            problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
+            with open(problem_file) as fp:
+                answers_with_file = {'1_2_1': 'String-based answer',
+                                     '1_3_1': ['answer1', 'answer2', 'answer3'],
+                                     '1_4_1': [fp, fp]}
+                answers_converted = convert_files_to_filenames(answers_with_file)
+                self.assertEquals(answers_converted['1_2_1'], 'String-based answer')
+                self.assertEquals(answers_converted['1_3_1'], ['answer1', 'answer2', 'answer3'])
+                self.assertEquals(answers_converted['1_4_1'], [fp.name, fp.name])
 
 
 class ChoiceResponseTest(unittest.TestCase):

From 6302283183fe8adff82aaa02a096be15814e47c5 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 19:57:46 -0400
Subject: [PATCH 054/181] Drop unnecessary .keys()

---
 common/lib/xmodule/xmodule/tests/__init__.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/common/lib/xmodule/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py
index 5385b489b2..4c97710607 100644
--- a/common/lib/xmodule/xmodule/tests/__init__.py
+++ b/common/lib/xmodule/xmodule/tests/__init__.py
@@ -298,7 +298,7 @@ class CodeResponseTest(unittest.TestCase):
         with open(problem_file) as input_file:
             test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
             
-            answer_ids = sorted(test_lcp.get_question_answers().keys())
+            answer_ids = sorted(test_lcp.get_question_answers())
             num_answers = len(answer_ids)
 
             # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
@@ -326,7 +326,7 @@ class CodeResponseTest(unittest.TestCase):
         with open(problem_file) as input_file:
             test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
 
-            answer_ids = sorted(test_lcp.get_question_answers().keys())
+            answer_ids = sorted(test_lcp.get_question_answers())
             num_answers = len(answer_ids)
 
             # CodeResponse requires internal CorrectMap state. Build it now in the queued state
@@ -383,7 +383,7 @@ class CodeResponseTest(unittest.TestCase):
         with open(problem_file) as input_file:
             test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
 
-            answer_ids = sorted(test_lcp.get_question_answers().keys())
+            answer_ids = sorted(test_lcp.get_question_answers())
             num_answers = len(answer_ids)
 
             # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
@@ -396,7 +396,6 @@ class CodeResponseTest(unittest.TestCase):
 
             # CodeResponse requires internal CorrectMap state. Build it now in the queued state
             cmap = CorrectMap()
-            answer_ids = sorted(test_lcp.get_question_answers().keys())
             num_answers = len(answer_ids)
             for i in range(num_answers):
                 queuekey = 1000 + i

From 297df37fc05bfef4f674bc4661cacbe09d6969e1 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 19:59:09 -0400
Subject: [PATCH 055/181] Drop unnecessary iterator index

---
 common/lib/xmodule/xmodule/tests/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/common/lib/xmodule/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py
index 4c97710607..b19b7e012b 100644
--- a/common/lib/xmodule/xmodule/tests/__init__.py
+++ b/common/lib/xmodule/xmodule/tests/__init__.py
@@ -303,8 +303,8 @@ class CodeResponseTest(unittest.TestCase):
 
             # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
             cmap = CorrectMap()
-            for i in range(num_answers):
-                cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=None))
+            for answer_id in answer_ids:
+                cmap.update(CorrectMap(answer_id=answer_id, queuestate=None))
             test_lcp.correct_map.update(cmap)
 
             self.assertEquals(test_lcp.is_queued(), False)

From 82a0d065a76e644fe92723cae8c1fa9929a1c138 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 20:04:48 -0400
Subject: [PATCH 056/181] Use enumerate in loops

---
 common/lib/xmodule/xmodule/tests/__init__.py | 32 +++++++++-----------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/common/lib/xmodule/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py
index b19b7e012b..a4c02a17f4 100644
--- a/common/lib/xmodule/xmodule/tests/__init__.py
+++ b/common/lib/xmodule/xmodule/tests/__init__.py
@@ -299,7 +299,6 @@ class CodeResponseTest(unittest.TestCase):
             test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
             
             answer_ids = sorted(test_lcp.get_question_answers())
-            num_answers = len(answer_ids)
 
             # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
             cmap = CorrectMap()
@@ -311,7 +310,7 @@ class CodeResponseTest(unittest.TestCase):
 
             # Now we queue the LCP
             cmap = CorrectMap()
-            for i in range(num_answers):
+            for i, answer_id in enumerate(answer_ids): 
                 queuestate = CodeResponseTest.make_queuestate(i, datetime.now())
                 cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
             test_lcp.correct_map.update(cmap)
@@ -327,11 +326,10 @@ class CodeResponseTest(unittest.TestCase):
             test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
 
             answer_ids = sorted(test_lcp.get_question_answers())
-            num_answers = len(answer_ids)
 
             # CodeResponse requires internal CorrectMap state. Build it now in the queued state
             old_cmap = CorrectMap()
-            for i in range(num_answers):
+            for i, answer_id in enumerate(answer_ids):
                 queuekey = 1000 + i
                 queuestate = CodeResponseTest.make_queuestate(1000+i, datetime.now())
                 old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
@@ -351,28 +349,28 @@ class CodeResponseTest(unittest.TestCase):
                 test_lcp.update_score(xserver_msgs[correctness], queuekey=0)
                 self.assertEquals(test_lcp.correct_map.get_dict(), old_cmap.get_dict())  # Deep comparison
 
-                for i in range(num_answers):
-                    self.assertTrue(test_lcp.correct_map.is_queued(answer_ids[i]))  # Should be still queued, since message undelivered
+                for answer_id in answer_ids:
+                    self.assertTrue(test_lcp.correct_map.is_queued(answer_id))  # Should be still queued, since message undelivered
 
             # Correct queuekey, state should be updated
             for correctness in ['correct', 'incorrect']:
-                for i in range(num_answers):  # Target specific answer_id's
+                for i, answer_id in enumerate(answer_ids):
                     test_lcp.correct_map = CorrectMap()
                     test_lcp.correct_map.update(old_cmap)
 
                     new_cmap = CorrectMap()
                     new_cmap.update(old_cmap)
                     npoints = 1 if correctness=='correct' else 0
-                    new_cmap.set(answer_id=answer_ids[i], npoints=npoints, correctness=correctness, msg='MESSAGE', queuestate=None)
+                    new_cmap.set(answer_id=answer_id, npoints=npoints, correctness=correctness, msg='MESSAGE', queuestate=None)
 
                     test_lcp.update_score(xserver_msgs[correctness], queuekey=1000 + i)
                     self.assertEquals(test_lcp.correct_map.get_dict(), new_cmap.get_dict())
 
-                    for j in range(num_answers):
+                    for j, test_id in enumerate(answer_ids):
                         if j == i:
-                            self.assertFalse(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be dequeued, message delivered
+                            self.assertFalse(test_lcp.correct_map.is_queued(test_id))  # Should be dequeued, message delivered
                         else:
-                            self.assertTrue(test_lcp.correct_map.is_queued(answer_ids[j]))  # Should be queued, message undelivered
+                            self.assertTrue(test_lcp.correct_map.is_queued(test_id))  # Should be queued, message undelivered
     
 
     def test_recentmost_queuetime(self):
@@ -384,28 +382,26 @@ class CodeResponseTest(unittest.TestCase):
             test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=i4xs)
 
             answer_ids = sorted(test_lcp.get_question_answers())
-            num_answers = len(answer_ids)
 
             # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
             cmap = CorrectMap()
-            for i in range(num_answers):
-                cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=None))
+            for answer_id in answer_ids:
+                cmap.update(CorrectMap(answer_id=answer_id, queuestate=None))
             test_lcp.correct_map.update(cmap)
             
             self.assertEquals(test_lcp.get_recentmost_queuetime(), None)
 
             # CodeResponse requires internal CorrectMap state. Build it now in the queued state
             cmap = CorrectMap()
-            num_answers = len(answer_ids)
-            for i in range(num_answers):
+            for i, answer_id in enumerate(answer_ids):
                 queuekey = 1000 + i
                 latest_timestamp = datetime.now()
                 queuestate = CodeResponseTest.make_queuestate(1000+i, latest_timestamp)
-                cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
+                cmap.update(CorrectMap(answer_id=answer_id, queuestate=queuestate))
             test_lcp.correct_map.update(cmap)
 
             # Queue state only tracks up to second
-            latest_timestamp = datetime.strptime(datetime.strftime(latest_timestamp,'%Y%m%d%H%M%S'),'%Y%m%d%H%M%S')
+            latest_timestamp = datetime.strptime(datetime.strftime(latest_timestamp, dateformat), dateformat)
 
             self.assertEquals(test_lcp.get_recentmost_queuetime(), latest_timestamp)
 

From 377b09e170fc7aa21aef6a2b021580602d567cda Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Mon, 27 Aug 2012 20:07:54 -0400
Subject: [PATCH 057/181] Adjust comments

---
 common/lib/xmodule/xmodule/capa_module.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py
index 27641af3b5..cfe3d2d48b 100644
--- a/common/lib/xmodule/xmodule/capa_module.py
+++ b/common/lib/xmodule/xmodule/capa_module.py
@@ -462,14 +462,14 @@ class CapaModule(XModule):
             self.system.track_function('save_problem_check_fail', event_info)
             raise NotFoundError('Problem must be reset before it can be checked again')
 
-        # Problem queued. Students must wait XQUEUE_WAITTIME_BETWEEN_REQUESTS
+        # Problem queued. Students must wait a specified waittime before they are allowed to submit
         if self.lcp.is_queued():
             current_time = datetime.datetime.now()
             prev_submit_time = self.lcp.get_recentmost_queuetime() 
             waittime_between_requests = self.system.xqueue['waittime']
             if (current_time-prev_submit_time).total_seconds() < waittime_between_requests:
                 msg = 'You must wait at least %d seconds between submissions' % waittime_between_requests
-                return {'success': msg, 'html': ''}
+                return {'success': msg, 'html': ''} # Prompts a modal dialog in ajax callback 
 
         try:
             old_state = self.lcp.get_state()

From 8ed9ab44e74c82cccda8d85546234be9c22d4db6 Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Mon, 27 Aug 2012 23:48:42 -0700
Subject: [PATCH 058/181] Contextualizing javascript parameters

---
 common/lib/capa/capa/responsetypes.py         | 38 ++++++++++---------
 .../xmodule/js/src/capa/display.coffee        |  3 ++
 2 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index b2d56b48ca..0e14ac46c1 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -384,19 +384,23 @@ class JavascriptResponse(LoncapaResponse):
         node_path = self.system.node_path + ":" + os.path.normpath(js_dir)
         tmp_env["NODE_PATH"] = node_path
         return tmp_env
+    
+    def call_node(self, args):
+
+        subprocess_args = ["node"]
+        subprocess_args.extend(args)
+
+        return subprocess.check_output(subprocess_args, env=self.get_node_env())
 
 
     def generate_problem_state(self):
 
         generator_file = os.path.dirname(os.path.normpath(__file__)) + '/javascript_problem_generator.js'
-        output = subprocess.check_output(["node",
-                                          generator_file, 
-                                          self.generator, 
-                                          json.dumps(self.generator_dependencies),
-                                          json.dumps(str(self.system.seed)), 
-                                          json.dumps(self.params)
-                                          ],
-                                          env=self.get_node_env()).strip()
+        output = self.call_node([generator_file,
+                                 self.generator, 
+                                 json.dumps(self.generator_dependencies),
+                                 json.dumps(str(self.system.seed)), 
+                                 json.dumps(self.params)]).strip()
 
         return json.loads(output)
 
@@ -407,7 +411,8 @@ class JavascriptResponse(LoncapaResponse):
         for param in self.xml.xpath('//*[@id=$id]//responseparam', 
                                         id=self.xml.get('id')):
 
-            params[param.get("name")] = json.loads(param.get("value"))
+            raw_param = param.get("value")
+            params[param.get("name")] = contextualize_text(raw_param, self.context)
         
         return params
 
@@ -442,15 +447,12 @@ class JavascriptResponse(LoncapaResponse):
             submission = json.dumps(None)
 
         grader_file = os.path.dirname(os.path.normpath(__file__)) + '/javascript_problem_grader.js'
-        outputs = subprocess.check_output(["node",
-                                           grader_file, 
-                                           self.grader, 
-                                           json.dumps(self.grader_dependencies),
-                                           submission, 
-                                           json.dumps(self.problem_state), 
-                                           json.dumps(self.params)
-                                          ],
-                                          env=self.get_node_env()).split('\n')
+        outputs = self.call_node([grader_file, 
+                                  self.grader, 
+                                  json.dumps(self.grader_dependencies),
+                                  submission, 
+                                  json.dumps(self.problem_state), 
+                                  json.dumps(self.params)]).split('\n')
 
         all_correct = json.loads(outputs[0].strip())
         evaluation  = outputs[1].strip()
diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee
index 23fa4d70fe..27d3585f96 100644
--- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee
+++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee
@@ -293,6 +293,9 @@ class @Problem
       problemState  = data.data("problem_state")
       displayClass  = window[data.data('display_class')]
 
+      if evaluation == ''
+          evaluation = null
+
       container = $(element).find(".javascriptinput_container")
       submissionField = $(element).find(".javascriptinput_input")
 

From 3842c39f8a3e5c7fc1c55681310cfc3ff1274e40 Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Mon, 27 Aug 2012 23:52:27 -0700
Subject: [PATCH 059/181] Fixing order that problem scores are displayed in

---
 lms/djangoapps/courseware/grades.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lms/djangoapps/courseware/grades.py b/lms/djangoapps/courseware/grades.py
index 1af3ab1bda..7f28f3ca5c 100644
--- a/lms/djangoapps/courseware/grades.py
+++ b/lms/djangoapps/courseware/grades.py
@@ -17,6 +17,7 @@ log = logging.getLogger("mitx.courseware")
 
 def yield_module_descendents(module):
     stack = module.get_display_items()
+    stack.reverse()
 
     while len(stack) > 0:
         next_module = stack.pop()

From f7ebc4dd23f1f6c0700902feaac7e68a2e98fc02 Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Tue, 28 Aug 2012 00:49:32 -0700
Subject: [PATCH 060/181] Load params from json

---
 common/lib/capa/capa/responsetypes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index 0e14ac46c1..4d24fc6c23 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -412,7 +412,7 @@ class JavascriptResponse(LoncapaResponse):
                                         id=self.xml.get('id')):
 
             raw_param = param.get("value")
-            params[param.get("name")] = contextualize_text(raw_param, self.context)
+            params[param.get("name")] = json.loads(contextualize_text(raw_param, self.context))
         
         return params
 

From 02e076b7250ecf09837428a4812dc8b890c35d5f Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Tue, 28 Aug 2012 05:49:36 -0700
Subject: [PATCH 061/181] adding pygraphviz

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 72b13e63ba..3376fc1a1d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -46,4 +46,5 @@ django-sekizai<0.7
 django-mptt>=0.5.3
 sorl-thumbnail
 networkx
+pygraphviz
 -r repo-requirements.txt

From ad745aa9bdfc0589631d2a4693a2818a5dc1d1b7 Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Tue, 28 Aug 2012 05:53:33 -0700
Subject: [PATCH 062/181] pygraphviz requires libgraphviz-dev on ubuntu

---
 create-dev-env.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/create-dev-env.sh b/create-dev-env.sh
index 96f212c9b5..3664129775 100755
--- a/create-dev-env.sh
+++ b/create-dev-env.sh
@@ -105,7 +105,7 @@ NUMPY_VER="1.6.2"
 SCIPY_VER="0.10.1"
 BREW_FILE="$BASE/mitx/brew-formulas.txt"
 LOG="/var/tmp/install-$(date +%Y%m%d-%H%M%S).log"
-APT_PKGS="curl git python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript graphviz"
+APT_PKGS="curl git python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript graphviz libgraphviz-dev"
 
 if [[ $EUID -eq 0 ]]; then
     error "This script should not be run using sudo or as the root user"

From 947ec74afc0cdc76b594d25edb6214c7ce912724 Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Tue, 28 Aug 2012 06:13:41 -0700
Subject: [PATCH 063/181] Let JS responses return >1 point

---
 common/lib/capa/capa/responsetypes.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index b2d56b48ca..d721322d6b 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -435,7 +435,11 @@ class JavascriptResponse(LoncapaResponse):
         (all_correct, evaluation, solution) = self.run_grader(json_submission)
         self.solution = solution
         correctness = 'correct' if all_correct else 'incorrect'
-        return CorrectMap(self.answer_id, correctness, msg=evaluation)
+        if all_correct:
+            points = self.get_max_score()
+        else:
+            points = 0
+        return CorrectMap(self.answer_id, correctness, npoints=points, msg=evaluation)
     
     def run_grader(self, submission):
         if submission is None or submission == '':

From 024b9db10dfb0d462488cdda9f252ef2fd8f9417 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Tue, 28 Aug 2012 09:27:27 -0400
Subject: [PATCH 064/181] Log errors in external grader message parsing

---
 common/lib/capa/capa/responsetypes.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index e7d7406cbd..620152be57 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -1210,13 +1210,13 @@ class CodeResponse(LoncapaResponse):
         try:
             score_result = json.loads(score_msg)
         except (TypeError, ValueError):
-            log.error("External grader message should be a JSON-serialized dict")
+            log.error("External grader message should be a JSON-serialized dict. Received score_msg = %s" % score_msg)
             return fail
         if not isinstance(score_result, dict):
-            log.error("External grader message should be a JSON-serialized dict")
+            log.error("External grader message should be a JSON-serialized dict. Received score_result = %s" % score_result)
             return fail
         for tag in ['correct', 'score', 'msg']:
-            if not score_result.has_key(tag):
+            if tag not in score_result:
                 log.error("External grader message is missing one or more required tags: 'correct', 'score', 'msg'")
                 return fail
 

From bd2374b6fe6dd0ddbecc460f2166891a76c4ee23 Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Tue, 28 Aug 2012 10:22:08 -0400
Subject: [PATCH 065/181] Remove trailing slash from ajax url in ModuleSystem

* also add a check for valid location in modx_dispatch
---
 lms/djangoapps/courseware/module_render.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py
index 6c3db9bc18..e238e9ca06 100644
--- a/lms/djangoapps/courseware/module_render.py
+++ b/lms/djangoapps/courseware/module_render.py
@@ -198,6 +198,8 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
                                    location=descriptor.location.url(),
                                    dispatch=''),
                        )
+    # Intended use is as {ajax_url}/{dispatch_command}, so get rid of the trailing slash.
+    ajax_url = ajax_url.rstrip('/')
 
     # Fully qualified callback URL for external queueing system
     xqueue_callback_url = '{proto}://{host}'.format(
@@ -410,6 +412,10 @@ def modx_dispatch(request, dispatch, location, course_id):
     '''
     # ''' (fix emacs broken parsing)
 
+    # Check parameters and fail fast if there's a problem
+    if not Location.is_valid(location):
+        raise Http404("Invalid location")
+
     # Check for submitted files and basic file size checks
     p = request.POST.copy()
     if request.FILES:

From 9277f5df8eeaa7a13ed6f54a256d90716dc8f5e6 Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Tue, 28 Aug 2012 11:27:47 -0400
Subject: [PATCH 066/181] Minor cleanups to url name checking

* better error messages
* better fallback names
* architectural TODO for later...
---
 common/lib/xmodule/xmodule/modulestore/xml.py | 27 ++++++++++++-------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py
index 92eca8f5e6..b0910efca9 100644
--- a/common/lib/xmodule/xmodule/modulestore/xml.py
+++ b/common/lib/xmodule/xmodule/modulestore/xml.py
@@ -79,11 +79,12 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                             del attr[key]
                         break
 
-                def fallback_name():
+                def fallback_name(orig_name=None):
                     """Return the fallback name for this module.  This is a function instead of a variable
                     because we want it to be lazy."""
-                    # use the hash of the content--the first 12 bytes should be plenty.
-                    return tag + "_" + hashlib.sha1(xml).hexdigest()[:12]
+                    # append the hash of the content--the first 12 bytes should be plenty.
+                    orig_name = "_" + orig_name if orig_name is not None else ""
+                    return tag + orig_name + "_" + hashlib.sha1(xml).hexdigest()[:12]
 
                 # Fallback if there was nothing we could use:
                 if url_name is None or url_name == "":
@@ -93,8 +94,9 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                     need_uniq_names = ('problem', 'sequence', 'video', 'course', 'chapter')
 
                     if tag in need_uniq_names:
-                        error_tracker("ERROR: no name of any kind specified for {tag}.  Student "
-                                      "state won't work right.  Problem xml: '{xml}...'".format(tag=tag, xml=xml[:100]))
+                        error_tracker("PROBLEM: no name of any kind specified for {tag}.  Student "
+                                      "state will not be properly tracked for this module.  Problem xml:"
+                                      " '{xml}...'".format(tag=tag, xml=xml[:100]))
                     else:
                         # TODO (vshnayder): We may want to enable this once course repos are cleaned up.
                         # (or we may want to give up on the requirement for non-state-relevant issues...)
@@ -103,13 +105,20 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
 
                 # Make sure everything is unique
                 if url_name in self.used_names[tag]:
-                    msg = ("Non-unique url_name in xml.  This may break content.  url_name={0}.  Content={1}"
-                                .format(url_name, xml[:100]))
-                    error_tracker("ERROR: " + msg)
+                    msg = ("Non-unique url_name in xml.  This may break state tracking for content."
+                           "  url_name={0}.  Content={1}".format(url_name, xml[:100]))
+                    error_tracker("PROBLEM: " + msg)
                     log.warning(msg)
                     # Just set name to fallback_name--if there are multiple things with the same fallback name,
                     # they are actually identical, so it's fragile, but not immediately broken.
-                    url_name = fallback_name()
+
+                    # TODO (vshnayder): if the tag is a pointer tag, this will
+                    # break the content because we won't have the right link.
+                    # That's also a legitimate attempt to reuse the same content
+                    # from multiple places.  Once we actually allow that, we'll
+                    # need to update this to complain about non-unique names for
+                    # definitions, but allow multiple uses.
+                    url_name = fallback_name(url_name)
 
                 self.used_names[tag].add(url_name)
                 xml_data.set('url_name', url_name)

From e9ded1cf3639c21e283761a602e445566440937a Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Tue, 28 Aug 2012 11:28:10 -0400
Subject: [PATCH 067/181] slight refactoring to test file

* was going to use to test duplicated content, but we're not supporting that for now
---
 common/test/data/full/chapter/Overview.xml | 2 +-
 common/test/data/full/video/welcome.xml    | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 common/test/data/full/video/welcome.xml

diff --git a/common/test/data/full/chapter/Overview.xml b/common/test/data/full/chapter/Overview.xml
index 89917d20da..a11a11a1e0 100644
--- a/common/test/data/full/chapter/Overview.xml
+++ b/common/test/data/full/chapter/Overview.xml
@@ -1,5 +1,5 @@
 <sequential>
-  <video youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8" slug="Welcome" format="Video" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="Welcome"/>
+  <video url_name="welcome"/>
   <sequential filename="System_Usage_Sequence" slug="System_Usage_Sequence" format="Lecture Sequence" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="System Usage Sequence"/>
   <vertical slug="Lab0_Using_the_tools" format="Lab" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="Lab0: Using the tools">
     <html slug="html_19" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab </html>
diff --git a/common/test/data/full/video/welcome.xml b/common/test/data/full/video/welcome.xml
new file mode 100644
index 0000000000..762bbeeaf1
--- /dev/null
+++ b/common/test/data/full/video/welcome.xml
@@ -0,0 +1 @@
+<video youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8" format="Video" display_name="Welcome"/>

From 5c17366cd54fe9ef0552d78e4ace94bc0f01d82a Mon Sep 17 00:00:00 2001
From: Calen Pennington <calen.pennington@gmail.com>
Date: Tue, 28 Aug 2012 12:01:05 -0400
Subject: [PATCH 068/181] Add a wsgi file for the lms that loads the
 modulestores immediately after the application starts

---
 lms/wsgi.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 lms/wsgi.py

diff --git a/lms/wsgi.py b/lms/wsgi.py
new file mode 100644
index 0000000000..270b019add
--- /dev/null
+++ b/lms/wsgi.py
@@ -0,0 +1,14 @@
+import os
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "lms.envs.aws")
+
+# This application object is used by the development server
+# as well as any WSGI server configured to use this file.
+from django.core.wsgi import get_wsgi_application
+application = get_wsgi_application()
+
+from django.conf import settings
+from xmodule.modulestore.django import modulestore
+
+for store_name in settings.MODULESTORE:
+    modulestore(store_name)

From 44cff00f50d0807c1777358e65cbaff131853067 Mon Sep 17 00:00:00 2001
From: Sarina Canelake <sarina@mitx.mit.edu>
Date: Tue, 28 Aug 2012 12:20:36 -0400
Subject: [PATCH 069/181] Fix display titles on video pages

---
 lms/templates/video.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lms/templates/video.html b/lms/templates/video.html
index 93273ddb87..fcf25f3d69 100644
--- a/lms/templates/video.html
+++ b/lms/templates/video.html
@@ -1,5 +1,5 @@
-% if name is not UNDEFINED and name is not None:
-  <h1> ${display_name} </h1>
+% if display_name is not UNDEFINED and display_name is not None:
+  <h2> ${display_name} </h2>
 % endif
 
 <div id="video_${id}" class="video" data-streams="${streams}" data-caption-data-dir="${data_dir}">

From eb48cae8875692cdd3a0608ce3145b0d2f265887 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Tue, 28 Aug 2012 12:50:48 -0400
Subject: [PATCH 070/181] Consolidate the timestamp uses in CodeResponse

---
 common/lib/capa/capa/responsetypes.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index e10c124d55..7bd9106b88 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -1127,11 +1127,14 @@ class CodeResponse(LoncapaResponse):
 
         # Prepare xqueue request
         #------------------------------------------------------------ 
+
         qinterface = self.system.xqueue['interface']
+        qtime = datetime.strftime(datetime.now(), xqueue_interface.dateformat)
+
         anonymous_student_id = self.system.anonymous_student_id
 
         # Generate header
-        queuekey = xqueue_interface.make_hashkey(str(self.system.seed) + str(time.time()) +
+        queuekey = xqueue_interface.make_hashkey(str(self.system.seed) + qtime +
                                                  anonymous_student_id +  
                                                  self.answer_id)
         xheader = xqueue_interface.make_xheader(lms_callback_url=self.system.xqueue['callback_url'],
@@ -1147,9 +1150,8 @@ class CodeResponse(LoncapaResponse):
         contents = self.payload.copy() 
 
         # Metadata related to the student submission revealed to the external grader
-        current_time = datetime.now()
         student_info = {'anonymous_student_id': anonymous_student_id,
-                        'submission_time': str(current_time),
+                        'submission_time': qtime,
                        }
         contents.update({'student_info': json.dumps(student_info)})
 
@@ -1165,7 +1167,6 @@ class CodeResponse(LoncapaResponse):
                                                     body=json.dumps(contents))
 
         # State associated with the queueing request
-        qtime = datetime.strftime(datetime.now(), xqueue_interface.dateformat)
         queuestate = {'key': queuekey,
                       'time': qtime,
                      }

From 4970d9d9f9c331bcccd424170fb1329ab6c4b0f7 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Tue, 28 Aug 2012 12:55:04 -0400
Subject: [PATCH 071/181] Don't need time import, have datetime

---
 common/lib/capa/capa/responsetypes.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index 7bd9106b88..4b3cde9d73 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -16,7 +16,6 @@ import numpy
 import random
 import re
 import requests
-import time
 import traceback
 import hashlib
 import abc

From fa074277cf56b0138356ab8a09132e77e24da2ff Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Tue, 28 Aug 2012 12:59:30 -0400
Subject: [PATCH 072/181] Adjust CodeResponse comments

---
 common/lib/capa/capa/responsetypes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index 4b3cde9d73..c5c6c895a4 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -1010,7 +1010,7 @@ class CodeResponse(LoncapaResponse):
     '''
     Grade student code using an external queueing server, called 'xqueue'
 
-    Expects 'xqueue' dict in ModuleSystem with the following keys:
+    Expects 'xqueue' dict in ModuleSystem with the following keys that are needed by CodeResponse:
         system.xqueue = { 'interface': XqueueInterface object,
                           'callback_url': Per-StudentModule callback URL where results are posted (string),
                           'default_queuename': Default queuename to submit request (string)

From bad24df84274e6be75fa0b38cae0a21bcc7a2f4b Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Tue, 28 Aug 2012 13:31:06 -0400
Subject: [PATCH 073/181] Adjust tests to reflect XML requirement for external
 grader messages

---
 common/lib/xmodule/xmodule/tests/__init__.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/common/lib/xmodule/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py
index 7697c96214..f95a92397a 100644
--- a/common/lib/xmodule/xmodule/tests/__init__.py
+++ b/common/lib/xmodule/xmodule/tests/__init__.py
@@ -285,6 +285,7 @@ class StringResponseWithHintTest(unittest.TestCase):
 class CodeResponseTest(unittest.TestCase):
     '''
     Test CodeResponse
+    TODO: Add tests for external grader messages
     '''
     @staticmethod
     def make_queuestate(key, time):
@@ -318,6 +319,7 @@ class CodeResponseTest(unittest.TestCase):
 
             self.assertEquals(test_lcp.is_queued(), True)
 
+
     def test_update_score(self):
         '''
         Test whether LoncapaProblem.update_score can deliver queued result to the right subproblem
@@ -336,8 +338,9 @@ class CodeResponseTest(unittest.TestCase):
                 old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
 
             # Message format common to external graders
-            correct_score_msg = json.dumps({'correct':True, 'score':1, 'msg':'MESSAGE'})
-            incorrect_score_msg = json.dumps({'correct':False, 'score':0, 'msg':'MESSAGE'})
+            grader_msg = '<span>MESSAGE</span>' # Must be valid XML
+            correct_score_msg = json.dumps({'correct':True, 'score':1, 'msg': grader_msg})
+            incorrect_score_msg = json.dumps({'correct':False, 'score':0, 'msg': grader_msg})
 
             xserver_msgs = {'correct': correct_score_msg,
                             'incorrect': incorrect_score_msg,}
@@ -362,7 +365,7 @@ class CodeResponseTest(unittest.TestCase):
                     new_cmap = CorrectMap()
                     new_cmap.update(old_cmap)
                     npoints = 1 if correctness=='correct' else 0
-                    new_cmap.set(answer_id=answer_id, npoints=npoints, correctness=correctness, msg='MESSAGE', queuestate=None)
+                    new_cmap.set(answer_id=answer_id, npoints=npoints, correctness=correctness, msg=grader_msg, queuestate=None)
 
                     test_lcp.update_score(xserver_msgs[correctness], queuekey=1000 + i)
                     self.assertEquals(test_lcp.correct_map.get_dict(), new_cmap.get_dict())

From 6a6b24af79bc7da0806225a08b31fe063d1ca79c Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Tue, 28 Aug 2012 12:40:49 -0700
Subject: [PATCH 074/181] Disable jsresponse file compilation; done manually
 for now.

---
 common/lib/capa/capa/responsetypes.py         | 51 +++++++++++--------
 .../js/compiled/javascriptresponse.js         | 50 ++++++++++++++++++
 2 files changed, 79 insertions(+), 22 deletions(-)
 create mode 100644 common/lib/xmodule/xmodule/tests/test_files/js/compiled/javascriptresponse.js

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index 6655d3cb94..273b1a3f8f 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -318,30 +318,37 @@ class JavascriptResponse(LoncapaResponse):
     
     def compile_display_javascript(self):
 
-        latestTimestamp = 0
-        basepath = self.system.filestore.root_path + '/js/'
-        for filename in (self.display_dependencies + [self.display]):
-            filepath = basepath + filename
-            timestamp = os.stat(filepath).st_mtime
-            if timestamp > latestTimestamp:
-                latestTimestamp = timestamp
-        
-        h = hashlib.md5()
-        h.update(self.answer_id + str(self.display_dependencies))
-        compiled_filename = 'compiled/' + h.hexdigest() + '.js'
-        compiled_filepath = basepath + compiled_filename
+        # TODO FIXME
+        # arjun: removing this behavior for now (and likely forever). Keeping
+        # until we decide on exactly how to solve this issue. For now, files are
+        # manually being compiled to DATA_DIR/js/compiled.
 
-        if not os.path.exists(compiled_filepath) or os.stat(compiled_filepath).st_mtime < latestTimestamp:
-            outfile = open(compiled_filepath, 'w')
-            for filename in (self.display_dependencies + [self.display]):
-                filepath = basepath + filename
-                infile = open(filepath, 'r')
-                outfile.write(infile.read())
-                outfile.write(';\n')
-                infile.close()
-            outfile.close()
+        #latestTimestamp = 0
+        #basepath = self.system.filestore.root_path + '/js/'
+        #for filename in (self.display_dependencies + [self.display]):
+        #    filepath = basepath + filename
+        #    timestamp = os.stat(filepath).st_mtime
+        #    if timestamp > latestTimestamp:
+        #        latestTimestamp = timestamp
+        #
+        #h = hashlib.md5()
+        #h.update(self.answer_id + str(self.display_dependencies))
+        #compiled_filename = 'compiled/' + h.hexdigest() + '.js'
+        #compiled_filepath = basepath + compiled_filename
 
-        self.display_filename = compiled_filename
+        #if not os.path.exists(compiled_filepath) or os.stat(compiled_filepath).st_mtime < latestTimestamp:
+        #    outfile = open(compiled_filepath, 'w')
+        #    for filename in (self.display_dependencies + [self.display]):
+        #        filepath = basepath + filename
+        #        infile = open(filepath, 'r')
+        #        outfile.write(infile.read())
+        #        outfile.write(';\n')
+        #        infile.close()
+        #    outfile.close()
+
+        # TODO this should also be fixed when the above is fixed.
+        filename = self.system.ajax_url.split('/')[-1] + '.js'
+        self.display_filename = 'compiled/' + filename
 
     def parse_xml(self):
         self.generator_xml = self.xml.xpath('//*[@id=$id]//generator',
diff --git a/common/lib/xmodule/xmodule/tests/test_files/js/compiled/javascriptresponse.js b/common/lib/xmodule/xmodule/tests/test_files/js/compiled/javascriptresponse.js
new file mode 100644
index 0000000000..6670c6a09a
--- /dev/null
+++ b/common/lib/xmodule/xmodule/tests/test_files/js/compiled/javascriptresponse.js
@@ -0,0 +1,50 @@
+// Generated by CoffeeScript 1.3.3
+(function() {
+  var MinimaxProblemDisplay, root,
+    __hasProp = {}.hasOwnProperty,
+    __extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };
+
+  MinimaxProblemDisplay = (function(_super) {
+
+    __extends(MinimaxProblemDisplay, _super);
+
+    function MinimaxProblemDisplay(state, submission, evaluation, container, submissionField, parameters) {
+      this.state = state;
+      this.submission = submission;
+      this.evaluation = evaluation;
+      this.container = container;
+      this.submissionField = submissionField;
+      this.parameters = parameters != null ? parameters : {};
+      MinimaxProblemDisplay.__super__.constructor.call(this, this.state, this.submission, this.evaluation, this.container, this.submissionField, this.parameters);
+    }
+
+    MinimaxProblemDisplay.prototype.render = function() {};
+
+    MinimaxProblemDisplay.prototype.createSubmission = function() {
+      var id, value, _ref, _results;
+      this.newSubmission = {};
+      if (this.submission != null) {
+        _ref = this.submission;
+        _results = [];
+        for (id in _ref) {
+          value = _ref[id];
+          _results.push(this.newSubmission[id] = value);
+        }
+        return _results;
+      }
+    };
+
+    MinimaxProblemDisplay.prototype.getCurrentSubmission = function() {
+      return this.newSubmission;
+    };
+
+    return MinimaxProblemDisplay;
+
+  })(XProblemDisplay);
+
+  root = typeof exports !== "undefined" && exports !== null ? exports : this;
+
+  root.TestProblemDisplay = TestProblemDisplay;
+
+}).call(this);
+;

From aa063d57295464d3eb31463c89e5bb492a78771d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Andr=C3=A9s=20Rocha?= <rocha@edx.org>
Date: Tue, 28 Aug 2012 18:18:37 -0400
Subject: [PATCH 075/181] [#35065403] Lets video player continue to work if
 subtitles cannot be loaded

---
 .../js/src/video/display/video_caption.coffee | 52 +++++++++++--------
 1 file changed, 30 insertions(+), 22 deletions(-)

diff --git a/common/lib/xmodule/xmodule/js/src/video/display/video_caption.coffee b/common/lib/xmodule/xmodule/js/src/video/display/video_caption.coffee
index 1690fb6092..3094cc5119 100644
--- a/common/lib/xmodule/xmodule/js/src/video/display/video_caption.coffee
+++ b/common/lib/xmodule/xmodule/js/src/video/display/video_caption.coffee
@@ -1,4 +1,7 @@
 class @VideoCaption extends Subview
+  initialize: ->
+    @loaded = false  
+
   bind: ->
     $(window).bind('resize', @resize)
     @$('.hide-subtitles').click @toggle
@@ -24,6 +27,8 @@ class @VideoCaption extends Subview
       @captions = captions.text
       @start = captions.start
 
+      @loaded = true
+
       if onTouchBasedDevice()
         $('.subtitles li').html "Caption will be displayed when you start playing the video."
       else
@@ -47,37 +52,40 @@ class @VideoCaption extends Subview
     @rendered = true
 
   search: (time) ->
-    min = 0
-    max = @start.length - 1
+    if @loaded
+      min = 0
+      max = @start.length - 1
 
-    while min < max
-      index = Math.ceil((max + min) / 2)
-      if time < @start[index]
-        max = index - 1
-      if time >= @start[index]
-        min = index
-
-    return min
+      while min < max
+        index = Math.ceil((max + min) / 2)
+        if time < @start[index]
+          max = index - 1
+        if time >= @start[index]
+          min = index
+      return min
 
   play: ->
-    @renderCaption() unless @rendered
-    @playing = true
+    if @loaded
+      @renderCaption() unless @rendered
+      @playing = true
 
   pause: ->
-    @playing = false
+    if @loaded
+      @playing = false
 
   updatePlayTime: (time) ->
-    # This 250ms offset is required to match the video speed
-    time = Math.round(Time.convert(time, @currentSpeed, '1.0') * 1000 + 250)
-    newIndex = @search time
+    if @loaded
+      # This 250ms offset is required to match the video speed
+      time = Math.round(Time.convert(time, @currentSpeed, '1.0') * 1000 + 250)
+      newIndex = @search time
 
-    if newIndex != undefined && @currentIndex != newIndex
-      if @currentIndex
-        @$(".subtitles li.current").removeClass('current')
-      @$(".subtitles li[data-index='#{newIndex}']").addClass('current')
+      if newIndex != undefined && @currentIndex != newIndex
+        if @currentIndex
+          @$(".subtitles li.current").removeClass('current')
+        @$(".subtitles li[data-index='#{newIndex}']").addClass('current')
 
-      @currentIndex = newIndex
-      @scrollCaption()
+        @currentIndex = newIndex
+        @scrollCaption()
 
   resize: =>
     @$('.subtitles').css maxHeight: @captionHeight()

From cbfc117e8bdba6a62ee3ce638184489a4a8affe6 Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Sun, 26 Aug 2012 22:04:49 -0400
Subject: [PATCH 076/181] a few edits to the xml format docs

---
 doc/xml-format.md | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/doc/xml-format.md b/doc/xml-format.md
index 256d4839bf..3994a23c5d 100644
--- a/doc/xml-format.md
+++ b/doc/xml-format.md
@@ -148,7 +148,7 @@ That's basically all there is to the organizational structure.  Read the next se
 * `problem` -- a problem.  See elsewhere in edx4edx for documentation on the format.
 * `problemset` -- logically, a series of related problems.  Currently displayed vertically.  May contain explanatory html, videos, etc.
 * `sequential` -- a sequence of content, currently displayed with a horizontal list of tabs.  If possible, use a more semantically meaningful tag (currently, we only have `videosequence`).
-* `vertical` -- a sequence of content, displayed vertically.  If possible, use a more semantically meaningful tag (currently, we only have `problemset`).
+* `vertical` -- a sequence of content, displayed vertically.  Content will be accessed all at once, on the right part of the page. No navigational bar. May have to use browser scroll bars. Content split with separators.   If possible, use a more semantically meaningful tag (currently, we only have `problemset`).
 * `video`  -- a link to a video, currently expected to be hosted on youtube.
 * `videosequence` -- a sequence of videos.  This can contain various non-video content; it just signals to the system that this is logically part of an explanatory sequence of content, as opposed to say an exam sequence.
 
@@ -223,18 +223,23 @@ Values are dictionaries of the form {"metadata-key" : "metadata-value"}.
 __Not inherited:__
 
 * `display_name` - name that will appear when this content is displayed in the courseware.  Useful for all tag types.
-*	`format` - subheading under display name -- currently only displayed for chapter sub-sections.
+*	`format` - subheading under display name -- currently only displayed for chapter sub-sections.  Also used by the the grader to know how to process students assessments that the
+    section contains. New formats can be defined as a 'type' in the GRADER variable in course_settings.json. Optional.  (TODO: double check this--what's the current behavior?)
 * `hide_from_toc` -- If set to true for a chapter or chapter subsection, will hide that element from the courseware navigation accordion.  This is useful if you'd like to link to the content directly instead (e.g. for tutorials)
 * `ispublic` -- specify whether the course is public.  You should be able to use start dates instead (?)
 
 __Inherited:__
 
 * `start` -- when this content should be shown to students.  Note that anyone with staff access to the course will always see everything.
-*	`showanswer` - only for psets, is binary (closed/open).
-*	`graded` - Tutorial vs. grade, again binary (true/false). If true, will be used in calculation of student grade.
-*	`rerandomise` - Provide different numbers/variables for problems to prevent cheating. Provide different answers from questions bank?
-*	`due` - Due date for assignment. Assignment will be closed after that. This is a very important function of a policy file.
-* `graceperiod` -
+*	`showanswer` - When to show answer. For 'attempted', will show answer after first attempt. Values: never, attempted, answered, closed. Default: closed. Optional.
+*	`graded` - Whether this section will count towards the students grade. "true" or "false". Defaults to "false".
+*	`rerandomise` - Randomize question on each attempt. Values: 'always' (students see a different version of the problem after each attempt to solve it)
+                                                            'never' (all students see the same version of the problem)
+                                                            'per_student' (individual students see the same version of the problem each time the look at it, but that version is different from what other students see)
+                                                            Default: 'always'. Optional.
+*	`due` - Due date for assignment. Assignment will be closed after that.  Values: valid date. Default: none. Optional.
+* attempts: Number of allowed attempts. Values: integer. Default: infinite. Optional.
+* `graceperiod` - A default length of time that the problem is still accessible after the due date in the format "2 days 3 hours" or "1 day 15 minutes".  Note, graceperiods are currently the easiest way to handle time zones. Due dates are all expressed in UCT.
 * `xqa_key` -- for integration with Ike's content QA server. -- should typically be specified at the course level.
 
 __Inheritance example:__
@@ -258,6 +263,7 @@ Metadata can also live in the xml files, but anything defined in the policy file
    - note, some xml attributes are not metadata.  e.g. in `<video youtube="xyz987293487293847"/>`, the `youtube` attribute specifies what video this is, and is logically part of the content, not the policy, so it should stay in the xml.
 
 Another example policy file:
+
     {
         "course/2012": {
             "graceperiod": "1 day",
@@ -309,3 +315,7 @@ before the week 1 material to make it easy to find in the file.
 * A heads up: our content management system will allow you to develop content through a web browser, but will be backed by this same xml at first.  Once that happens, every element will be in its own file to make access and updates faster.
 
 * Prefer the most "semantic" name for containers: e.g., use problemset rather than vertical for a problem set.  That way, if we decide to display problem sets differently, we don't have to change the xml.
+
+----
+
+(Dev note: This file is generated from the mitx repo, in `doc/xml-format.md`.  Please make edits there.)

From fe50f9443781ded0903dcdfdb3f1fa469d4791aa Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Tue, 28 Aug 2012 19:51:25 -0400
Subject: [PATCH 077/181] Fix inline html bug.

* problem was in trying to remove the outer <html> + </html> tags.
* cleaned up the stringify function to be less magical, and more
  obviously correct.

[Fix #35136297]
---
 common/lib/xmodule/xmodule/stringify.py       | 16 +++++++---
 .../xmodule/xmodule/tests/test_stringify.py   | 31 ++++++++++++++++++-
 2 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/common/lib/xmodule/xmodule/stringify.py b/common/lib/xmodule/xmodule/stringify.py
index dad964140f..1e3fa91210 100644
--- a/common/lib/xmodule/xmodule/stringify.py
+++ b/common/lib/xmodule/xmodule/stringify.py
@@ -12,9 +12,17 @@ def stringify_children(node):
     fixed from
     http://stackoverflow.com/questions/4624062/get-all-text-inside-a-tag-in-lxml
     '''
-    parts = ([node.text] +
-            list(chain(*([etree.tostring(c), c.tail]
-                         for c in node.getchildren())
-                         )))
+    # Useful things to know:
+
+    # node.tostring() -- generates xml for the node, including start
+    #                 and end tags.  We'll use this for the children.
+    # node.text -- the text after the end of a start tag to the start
+    #                 of the first child
+    # node.tail -- the text after the end this tag to the start of the
+    #                 next element.
+    parts = [node.text]
+    for c in node.getchildren():
+        parts.append(etree.tostring(c, with_tail=True))
+
     # filter removes possible Nones in texts and tails
     return ''.join(filter(None, parts))
diff --git a/common/lib/xmodule/xmodule/tests/test_stringify.py b/common/lib/xmodule/xmodule/tests/test_stringify.py
index 1c6ee855f3..29e99bef56 100644
--- a/common/lib/xmodule/xmodule/tests/test_stringify.py
+++ b/common/lib/xmodule/xmodule/tests/test_stringify.py
@@ -1,4 +1,4 @@
-from nose.tools import assert_equals
+from nose.tools import assert_equals, assert_true, assert_false
 from lxml import etree
 from xmodule.stringify import stringify_children
 
@@ -8,3 +8,32 @@ def test_stringify():
     xml = etree.fromstring(html)
     out = stringify_children(xml)
     assert_equals(out, text)
+
+def test_stringify_again():
+    html = """<html name="Voltage Source Answer" >A voltage source is non-linear!
+<div align="center">
+    <img src="/static/images/circuits/voltage-source.png"/>
+    \(V=V_C\)
+  </div>
+  But it is <a href="http://mathworld.wolfram.com/AffineFunction.html">affine</a>,
+  which means linear except for an offset.
+  </html>
+"""
+
+    html = """<html>A voltage source is non-linear!
+  <div align="center">
+
+  </div>
+  But it is <a href="http://mathworld.wolfram.com/AffineFunction.html">affine</a>,
+  which means linear except for an offset.
+  </html>
+  """
+    xml = etree.fromstring(html)
+    out = stringify_children(xml)
+
+    print "output:"
+    print out
+
+    # Tracking strange content repeating bug
+    # Should appear once
+    assert_equals(out.count("But it is "), 1)

From ac65e47e21b12919b96aaf484f4ccdbe491b5328 Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Wed, 29 Aug 2012 03:15:30 -0700
Subject: [PATCH 078/181] Getting rid of an alert. People don't like alerts.

---
 common/lib/xmodule/xmodule/js/src/capa/display.coffee | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee
index 27d3585f96..ea5d939356 100644
--- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee
+++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee
@@ -257,7 +257,11 @@ class @Problem
     Logger.log 'problem_save', @answers
     $.postWithPrefix "#{@url}/problem_save", @answers, (response) =>
       if response.success
-        alert 'Saved'
+        if @el.find('.save_message').length
+            @el.find('.save_message').animate(opacity: 0, 500).animate(opacity: 1, 500)
+        else
+            saveMessage = "<div class='save_message'>Your answers have been saved but not graded. Hit 'Check' to grade them.</div>"
+            @el.find('.action').after(saveMessage)
       @updateProgress response
 
   refreshMath: (event, element) =>

From 1c176d369b9ff020e93bbc7244dfaa4f762363fd Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Wed, 29 Aug 2012 03:25:08 -0700
Subject: [PATCH 079/181] add margin between checkbox and text

---
 lms/static/sass/course/courseware/courses/_cs188.scss | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lms/static/sass/course/courseware/courses/_cs188.scss b/lms/static/sass/course/courseware/courses/_cs188.scss
index b069061e6b..07830e6a11 100644
--- a/lms/static/sass/course/courseware/courses/_cs188.scss
+++ b/lms/static/sass/course/courseware/courses/_cs188.scss
@@ -27,6 +27,12 @@ body.cs188 {
             margin-bottom: 1.416em;
         }
 
+        .choicegroup {
+            input[type=checkbox], input[type=radio] {
+                margin-right: 5px;
+            }
+        }
+
     }
 
 }

From 4b6694a4ced9510d131a73b74d7d5a8f7fe271ee Mon Sep 17 00:00:00 2001
From: Tommy MacWilliam <tmacwilliam@cs.harvard.edu>
Date: Wed, 15 Aug 2012 17:24:09 -0400
Subject: [PATCH 080/181] OpenID provider implementation

- endpoint supports both SReg and AX
- identity taken from edX username
- sreg fullname and ax http://axschema.org/namePerson taken from edX
  name
- sreg email and ax http://axschema.org/contact/email taken from edX
  email
---
 common/djangoapps/external_auth/views.py | 205 +++++++++++++++++++++++
 lms/templates/identity.xml               |  10 ++
 lms/templates/provider_login.html        |  35 ++++
 lms/templates/xrds.xml                   |  11 ++
 lms/urls.py                              |   4 +
 5 files changed, 265 insertions(+)
 create mode 100644 lms/templates/identity.xml
 create mode 100644 lms/templates/provider_login.html
 create mode 100644 lms/templates/xrds.xml

diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py
index 9e41d31c77..ece2fff00a 100644
--- a/common/djangoapps/external_auth/views.py
+++ b/common/djangoapps/external_auth/views.py
@@ -11,9 +11,12 @@ from django.conf import settings
 from django.contrib.auth import REDIRECT_FIELD_NAME, authenticate, login
 from django.contrib.auth.models import Group
 from django.contrib.auth.models import User
+from student.models import UserProfile
 
+from django.core.context_processors import csrf
 from django.core.urlresolvers import reverse
 from django.http import HttpResponse, HttpResponseRedirect
+from django.utils.http import urlquote
 from django.shortcuts import render_to_response
 from django.shortcuts import redirect
 from django.template import RequestContext
@@ -29,6 +32,14 @@ from django_openid_auth import auth as openid_auth
 from openid.consumer.consumer import (Consumer, SUCCESS, CANCEL, FAILURE)
 import django_openid_auth.views as openid_views
 
+from openid.server.server import Server, ProtocolError, CheckIDRequest, EncodingError
+from openid.server.trustroot import verifyReturnTo
+from openid.store.filestore import FileOpenIDStore
+from openid.yadis.discover import DiscoveryFailure
+from openid.consumer.discover import OPENID_IDP_2_0_TYPE
+from openid.extensions import ax, sreg
+from openid.fetchers import HTTPFetchingError
+
 import student.views as student_views
 
 log = logging.getLogger("mitx.external_auth")
@@ -218,3 +229,197 @@ def edXauth_ssl_login(request):
                                             email=email,
                                             fullname=fullname,
                                             retfun = functools.partial(student_views.index, request))
+
+def get_dict_for_openid(data):
+    """
+    Return a dictionary suitable for the OpenID library
+    """
+
+    return dict((k, v) for k, v in data.iteritems())
+
+def get_xrds_url(resource, request):
+    """
+    Return the XRDS url for a resource
+    """
+
+    location = request.META['HTTP_HOST'] + '/openid/provider/' + resource + '/'
+    if request.is_secure():
+        url = 'https://' + location
+    else:
+        url = 'http://' + location
+
+    return url
+
+def provider_respond(server, request, response, data):
+    """
+    Respond to an OpenID request
+    """
+
+    # get simple registration request
+    sreg_data = {}
+    sreg_request = sreg.SRegRequest.fromOpenIDRequest(request)
+    sreg_fields = sreg_request.allRequestedFields()
+
+    # if consumer requested simple registration fields, add them
+    if sreg_fields:
+        for field in sreg_fields:
+            if field == 'email' and 'email' in data:
+                sreg_data['email'] = data['email']
+            elif field == 'fullname' and 'fullname' in data:
+                sreg_data['fullname'] = data['fullname']
+
+        # construct sreg response
+        sreg_response = sreg.SRegResponse.extractResponse(sreg_request, sreg_data)
+        sreg_response.toMessage(response.fields)
+
+    # get attribute exchange request
+    try:
+        ax_request = ax.FetchRequest.fromOpenIDRequest(request)
+
+    except ax.AXError:
+        pass
+
+    else:
+        ax_response = ax.FetchResponse()
+
+        # if consumer requested attribute exchange fields, add them
+        if ax_request and ax_request.requested_attributes:
+            for type_uri in ax_request.requested_attributes.iterkeys():
+                if type_uri == 'http://axschema.org/contact/email' and 'email' in data:
+                    ax_response.addValue('http://axschema.org/contact/email', data['email'])
+
+                elif type_uri == 'http://axschema.org/namePerson' and 'fullname' in data:
+                    ax_response.addValue('http://axschema.org/namePerson', data['fullname']);
+
+            # construct ax response
+            ax_response.toMessage(response.fields)
+
+    # create http response from OpenID response
+    webresponse = server.encodeResponse(response)
+    http_response = HttpResponse(webresponse.body)
+    http_response.status_code = webresponse.code
+
+    # add OpenID headers to response
+    for k, v in webresponse.headers.iteritems():
+        http_response[k] = v
+
+    return http_response
+
+@csrf_exempt
+def provider_login(request):
+    """
+    OpenID login endpoint
+    """
+
+    # initialize store and server
+    endpoint = get_xrds_url('login', request)
+    store = FileOpenIDStore('/tmp/openid_provider')
+    server = Server(store, endpoint)
+
+    # handle OpenID request
+    query = get_dict_for_openid(request.GET or request.POST)
+    error = False
+    if 'openid.mode' in request.GET or 'openid.mode' in request.POST:
+        # decode request
+        openid_request = server.decodeRequest(query)
+
+        # checkid_immediate not supported, require user interaction
+        if openid_request.mode == 'checkid_immediate':
+            return provider_respond(server, openid_request, openid_request.answer(false), {})
+
+        # checkid_setup, so display login page
+        elif openid_request.mode == 'checkid_setup':
+            if openid_request.idSelect():
+                # remember request and original path
+                request.session['openid_request'] = {
+                    'request': openid_request,
+                    'url': request.get_full_path()
+                }
+
+                # user failed login on previous attempt
+                if 'openid_error' in request.session:
+                    error = True
+                    del request.session['openid_error']
+
+        # OpenID response
+        else:
+            return provider_respond(server, openid_request, server.handleRequest(openid_request), {})
+
+    # handle login
+    if request.method == 'POST' and 'openid_request' in request.session:
+        # get OpenID request from session
+        openid_request = request.session['openid_request']
+        del request.session['openid_request']
+
+        # check if user with given email exists
+        email = request.POST['email']
+        password = request.POST['password']
+        try:
+            user = User.objects.get(email=email)
+        except User.DoesNotExist:
+            request.session['openid_error'] = True
+            log.warning("Login failed - Unknown user email: {0}".format(email))
+            return HttpResponseRedirect(openid_request['url'])
+
+        # attempt to authenticate user
+        username = user.username
+        user = authenticate(username=username, password=password)
+        if user is None:
+            request.session['openid_error'] = True
+            log.warning("Login failed - password for {0} is invalid".format(email))
+            return HttpResponseRedirect(openid_request['url'])
+
+        # authentication succeeded, so log user in
+        if user is not None and user.is_active:
+            # remove error from session since login succeeded
+            if 'openid_error' in request.session:
+                del request.session['openid_error']
+
+            # fullname field comes from user profile
+            profile = UserProfile.objects.get(user=user)
+
+            # redirect user to return_to location
+            response = openid_request['request'].answer(True, None, endpoint + urlquote(user.username))
+            return provider_respond(server, openid_request['request'], response, {
+                'fullname': profile.name,
+                'email': user.email
+            })
+
+        request.session['openid_error'] = True
+        log.warning("Login failed - Account not active for user {0}".format(username))
+        return HttpResponseRedirect(openid_request['url'])
+
+    # display login page
+    response = render_to_response('provider_login.html', {
+        'error': error
+    })
+
+    # custom XRDS header necessary for discovery process
+    response['X-XRDS-Location'] = get_xrds_url('xrds', request)
+    return response
+
+def provider_identity(request):
+    """
+    XRDS for identity discovery
+    """
+
+    response = render_to_response('identity.xml', {
+        'url': get_xrds_url('login', request)
+    }, mimetype='text/xml')
+
+    # custom XRDS header necessary for discovery process
+    response['X-XRDS-Location'] = get_xrds_url('identity', request)
+    return response
+
+def provider_xrds(request):
+    """
+    XRDS for endpoint discovery
+    """
+
+    response = render_to_response('xrds.xml', {
+        'url': get_xrds_url('login', request)
+    }, mimetype='text/xml')
+
+    # custom XRDS header necessary for discovery process
+    response['X-XRDS-Location'] = get_xrds_url('xrds', request)
+    return response
diff --git a/lms/templates/identity.xml b/lms/templates/identity.xml
new file mode 100644
index 0000000000..a925493c03
--- /dev/null
+++ b/lms/templates/identity.xml
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xrds:XRDS xmlns:xrds="xri://$xrds" xmlns="xri://$xrd*($v*2.0)">
+    <XRD>
+        <Service priority="0">
+            <Type>http://specs.openid.net/auth/2.0/signon</Type>
+            <Type>http://openid.net/signon/1.1</Type>
+            <URI>${url}</URI>
+        </Service>
+    </XRD>
+</xrds:XRDS>
diff --git a/lms/templates/provider_login.html b/lms/templates/provider_login.html
new file mode 100644
index 0000000000..89061a3459
--- /dev/null
+++ b/lms/templates/provider_login.html
@@ -0,0 +1,35 @@
+<%inherit file="main.html" />
+<%namespace name='static' file='static_content.html'/>
+
+<style>
+
+.openid-login {
+    display: block;
+    position: relative;
+    left: 0;
+    margin: 100px auto;
+    top: 0;
+}
+
+</style>
+
+<section id="login-modal" class="modal login-modal openid-login">
+  <div class="inner-wrapper">
+    <header>
+      <h2>Log In</h2>
+      <hr>
+    </header>
+    <form id="login_form" class="login_form" method="post" action="/openid/provider/login/">
+%if error:
+          <div id="login_error" class="modal-form-error" style="display: block;">Email or password is incorrect.</div>
+%endif
+      <label>E-mail</label>
+      <input type="text" name="email" placeholder="E-mail" tabindex="1" />
+      <label>Password</label>
+      <input type="password" name="password" placeholder="Password" tabindex="2" />
+      <div class="submit">
+        <input name="submit" type="submit" value="Access My Courses" tabindex="3" />
+      </div>
+    </form>
+  </div>
+</section>
diff --git a/lms/templates/xrds.xml b/lms/templates/xrds.xml
new file mode 100644
index 0000000000..2f7713bc8a
--- /dev/null
+++ b/lms/templates/xrds.xml
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xrds:XRDS xmlns:xrds="xri://$xrds" xmlns="xri://$xrd*($v*2.0)">
+    <XRD>
+        <Service priority="0">
+            <Type>http://specs.openid.net/auth/2.0/server</Type>
+            <Type>http://openid.net/sreg/1.0</Type>
+            <Type>http://openid.net/srv/ax/1.0</Type>
+            <URI>${url}</URI>
+        </Service>
+    </XRD>
+</xrds:XRDS>
diff --git a/lms/urls.py b/lms/urls.py
index 86d654eb40..540b713167 100644
--- a/lms/urls.py
+++ b/lms/urls.py
@@ -217,6 +217,10 @@ if settings.MITX_FEATURES.get('AUTH_USE_OPENID'):
         url(r'^openid/login/$', 'django_openid_auth.views.login_begin', name='openid-login'),
         url(r'^openid/complete/$', 'external_auth.views.edXauth_openid_login_complete', name='openid-complete'),
         url(r'^openid/logo.gif$', 'django_openid_auth.views.logo', name='openid-logo'),
+        url(r'^openid/provider/login/$', 'external_auth.views.provider_login', name='openid-provider-login'),
+        url(r'^openid/provider/login/(?:[\w%\. ]+)$', 'external_auth.views.provider_identity', name='openid-provider-login-identity'),
+        url(r'^openid/provider/identity/$', 'external_auth.views.provider_identity', name='openid-provider-identity'),
+        url(r'^openid/provider/xrds/$', 'external_auth.views.provider_xrds', name='openid-provider-xrds')
         )
 
 if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):

From 6de28fc5928283c9fa659d7fd0a15df4cc8dfdc6 Mon Sep 17 00:00:00 2001
From: Tommy MacWilliam <tmacwilliam@cs.harvard.edu>
Date: Wed, 15 Aug 2012 19:49:53 -0400
Subject: [PATCH 081/181] feature flag, consumer displayed, logging

- provider placed behind separate feature flag
- domain of openid consumer displayed in login page
- added logging for successful login, changed log messages to include
  "OpenID"
---
 common/djangoapps/external_auth/views.py | 16 ++++++++++++----
 lms/envs/common.py                       |  2 +-
 lms/envs/dev.py                          |  1 +
 lms/templates/provider_login.html        |  9 +++++++--
 lms/urls.py                              |  6 +++++-
 5 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py
index ece2fff00a..1309ac0b7f 100644
--- a/common/djangoapps/external_auth/views.py
+++ b/common/djangoapps/external_auth/views.py
@@ -317,7 +317,7 @@ def provider_login(request):
     server = Server(store, endpoint)
 
     # handle OpenID request
-    query = get_dict_for_openid(request.GET or request.POST)
+    query = get_dict_for_openid(request.REQUEST)
     error = False
     if 'openid.mode' in request.GET or 'openid.mode' in request.POST:
         # decode request
@@ -358,7 +358,7 @@ def provider_login(request):
             user = User.objects.get(email=email)
         except User.DoesNotExist:
             request.session['openid_error'] = True
-            log.warning("Login failed - Unknown user email: {0}".format(email))
+            log.warning("OpenID login failed - Unknown user email: {0}".format(email))
             return HttpResponseRedirect(openid_request['url'])
 
         # attempt to authenticate user
@@ -366,7 +366,7 @@ def provider_login(request):
         user = authenticate(username=username, password=password)
         if user is None:
             request.session['openid_error'] = True
-            log.warning("Login failed - password for {0} is invalid".format(email))
+            log.warning("OpenID login failed - password for {0} is invalid".format(email))
             return HttpResponseRedirect(openid_request['url'])
 
         # authentication succeeded, so log user in
@@ -377,6 +377,7 @@ def provider_login(request):
 
             # fullname field comes from user profile
             profile = UserProfile.objects.get(user=user)
+            log.info("OpenID login success - {0} ({1})".format(user.username, user.email))
 
             # redirect user to return_to location
             response = openid_request['request'].answer(True, None, endpoint + urlquote(user.username))
@@ -389,9 +390,16 @@ def provider_login(request):
         log.warning("Login failed - Account not active for user {0}".format(username))
         return HttpResponseRedirect(openid_request['url'])
 
+    # determine consumer domain if applicable
+    return_to = ''
+    if 'openid.return_to' in request.REQUEST:
+        matches = re.match(r'\w+:\/\/([\w\.-]+)', request.REQUEST['openid.return_to'])
+        return_to = matches.group(1)
+
     # display login page
     response = render_to_response('provider_login.html', {
-        'error': error
+        'error': error,
+        'return_to': return_to
     })
 
     # custom XRDS header necessary for discovery process
diff --git a/lms/envs/common.py b/lms/envs/common.py
index ce08bf9666..9825df338d 100644
--- a/lms/envs/common.py
+++ b/lms/envs/common.py
@@ -77,7 +77,7 @@ MITX_FEATURES = {
     'ACCESS_REQUIRE_STAFF_FOR_COURSE': False,
     'AUTH_USE_OPENID': False,
     'AUTH_USE_MIT_CERTIFICATES' : False,
-
+    'AUTH_USE_OPENID_PROVIDER': False,
 }
 
 # Used for A/B testing
diff --git a/lms/envs/dev.py b/lms/envs/dev.py
index 97803cb47d..1b5da7ff71 100644
--- a/lms/envs/dev.py
+++ b/lms/envs/dev.py
@@ -17,6 +17,7 @@ MITX_FEATURES['DISABLE_START_DATES'] = True
 MITX_FEATURES['ENABLE_SQL_TRACKING_LOGS'] = True
 MITX_FEATURES['SUBDOMAIN_COURSE_LISTINGS'] = False  # Enable to test subdomains--otherwise, want all courses to show up
 MITX_FEATURES['SUBDOMAIN_BRANDING'] = True
+MITX_FEATURES['AUTH_USE_OPENID_PROVIDER'] = True
 
 WIKI_ENABLED = True
 
diff --git a/lms/templates/provider_login.html b/lms/templates/provider_login.html
index 89061a3459..73d35c019b 100644
--- a/lms/templates/provider_login.html
+++ b/lms/templates/provider_login.html
@@ -11,6 +11,11 @@
     top: 0;
 }
 
+.openid-login input[type=submit] {
+    white-space: normal;
+    height: auto !important;
+}
+
 </style>
 
 <section id="login-modal" class="modal login-modal openid-login">
@@ -21,14 +26,14 @@
     </header>
     <form id="login_form" class="login_form" method="post" action="/openid/provider/login/">
 %if error:
-          <div id="login_error" class="modal-form-error" style="display: block;">Email or password is incorrect.</div>
+      <div id="login_error" class="modal-form-error" style="display: block;">Email or password is incorrect.</div>
 %endif
       <label>E-mail</label>
       <input type="text" name="email" placeholder="E-mail" tabindex="1" />
       <label>Password</label>
       <input type="password" name="password" placeholder="Password" tabindex="2" />
       <div class="submit">
-        <input name="submit" type="submit" value="Access My Courses" tabindex="3" />
+        <input name="submit" type="submit" value="Access My Courses and Return To ${return_to}" tabindex="3" />
       </div>
     </form>
   </div>
diff --git a/lms/urls.py b/lms/urls.py
index 540b713167..380d168f62 100644
--- a/lms/urls.py
+++ b/lms/urls.py
@@ -217,11 +217,15 @@ if settings.MITX_FEATURES.get('AUTH_USE_OPENID'):
         url(r'^openid/login/$', 'django_openid_auth.views.login_begin', name='openid-login'),
         url(r'^openid/complete/$', 'external_auth.views.edXauth_openid_login_complete', name='openid-complete'),
         url(r'^openid/logo.gif$', 'django_openid_auth.views.logo', name='openid-logo'),
+    )
+
+if settings.MITX_FEATURES.get('AUTH_USE_OPENID_PROVIDER'):
+    urlpatterns += (
         url(r'^openid/provider/login/$', 'external_auth.views.provider_login', name='openid-provider-login'),
         url(r'^openid/provider/login/(?:[\w%\. ]+)$', 'external_auth.views.provider_identity', name='openid-provider-login-identity'),
         url(r'^openid/provider/identity/$', 'external_auth.views.provider_identity', name='openid-provider-identity'),
         url(r'^openid/provider/xrds/$', 'external_auth.views.provider_xrds', name='openid-provider-xrds')
-        )
+    )
 
 if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):
     urlpatterns += (

From 4126f3a29a8a66115944a462bf159daae3262237 Mon Sep 17 00:00:00 2001
From: Nate Hardison <natehardison@gmail.com>
Date: Wed, 29 Aug 2012 11:57:41 -0400
Subject: [PATCH 082/181] Validating OpenID request trust roots to only come
 from *.cs50.net

---
 common/djangoapps/external_auth/views.py | 44 ++++++++++++++++++++++--
 1 file changed, 42 insertions(+), 2 deletions(-)

diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py
index 1309ac0b7f..8c44182c48 100644
--- a/common/djangoapps/external_auth/views.py
+++ b/common/djangoapps/external_auth/views.py
@@ -33,7 +33,7 @@ from openid.consumer.consumer import (Consumer, SUCCESS, CANCEL, FAILURE)
 import django_openid_auth.views as openid_views
 
 from openid.server.server import Server, ProtocolError, CheckIDRequest, EncodingError
-from openid.server.trustroot import verifyReturnTo
+from openid.server.trustroot import TrustRoot
 from openid.store.filestore import FileOpenIDStore
 from openid.yadis.discover import DiscoveryFailure
 from openid.consumer.discover import OPENID_IDP_2_0_TYPE
@@ -255,7 +255,7 @@ def provider_respond(server, request, response, data):
     Respond to an OpenID request
     """
 
-    # get simple registration request
+    # get simple registration request 
     sreg_data = {}
     sreg_request = sreg.SRegRequest.fromOpenIDRequest(request)
     sreg_fields = sreg_request.allRequestedFields()
@@ -305,6 +305,37 @@ def provider_respond(server, request, response, data):
 
     return http_response
 
+
+def validate_trust_root(openid_request):
+    """
+    Only allow OpenID requests from valid trust roots
+    """
+
+    # verify the trust root/return to
+    trust_root = openid_request.trust_root
+    return_to = openid_request.return_to
+
+    # don't allow empty trust roots
+    if openid_request.trust_root is None:
+        return false 
+
+    # ensure trust root parses cleanly (one wildcard, of form *.foo.com, etc.)
+    trust_root = TrustRoot.parse(openid_request.trust_root) 
+    if trust_root is None:
+        return false
+
+    # don't allow empty return tos
+    if openid_request.return_to is None:
+        return false
+
+    # ensure return to is within trust root
+    if not trust_root.validateURL(openid_request.return_to):
+        return false
+
+    # only allow *.cs50.net for now
+    return trust_root.host.endswith('cs50.net')
+
+
 @csrf_exempt
 def provider_login(request):
     """
@@ -323,6 +354,10 @@ def provider_login(request):
         # decode request
         openid_request = server.decodeRequest(query)
 
+        # don't allow invalid and non-*.cs50.net trust roots
+        if not validate_trust_root(openid_request):
+            return default_render_failure(request, "Invalid OpenID trust root") 
+
         # checkid_immediate not supported, require user interaction
         if openid_request.mode == 'checkid_immediate':
             return provider_respond(server, openid_request, openid_request.answer(false), {})
@@ -351,6 +386,10 @@ def provider_login(request):
         openid_request = request.session['openid_request']
         del request.session['openid_request']
 
+        # don't allow invalid and non-*.cs50.net trust roots
+        if not validate_trust_root(openid_request):
+            return default_render_failure(request, "Invalid OpenID trust root") 
+
         # check if user with given email exists
         email = request.POST['email']
         password = request.POST['password']
@@ -431,3 +470,4 @@ def provider_xrds(request):
     # custom XRDS header necessary for discovery process
     response['X-XRDS-Location'] = get_xrds_url('xrds', request)
     return response
+

From cb67af59592582a298a04087bb150fd0c51208dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Andr=C3=A9s=20Rocha?= <rocha@edx.org>
Date: Wed, 29 Aug 2012 12:43:08 -0400
Subject: [PATCH 083/181] [34078525] OpenID provider cleanup and minor fixes

---
 common/djangoapps/external_auth/views.py | 221 ++++++++++++++---------
 1 file changed, 131 insertions(+), 90 deletions(-)

diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py
index 8c44182c48..149a185ee7 100644
--- a/common/djangoapps/external_auth/views.py
+++ b/common/djangoapps/external_auth/views.py
@@ -9,17 +9,12 @@ from external_auth.models import ExternalAuthMap
 
 from django.conf import settings
 from django.contrib.auth import REDIRECT_FIELD_NAME, authenticate, login
-from django.contrib.auth.models import Group
 from django.contrib.auth.models import User
 from student.models import UserProfile
 
-from django.core.context_processors import csrf
-from django.core.urlresolvers import reverse
 from django.http import HttpResponse, HttpResponseRedirect
 from django.utils.http import urlquote
-from django.shortcuts import render_to_response
 from django.shortcuts import redirect
-from django.template import RequestContext
 from mitxmako.shortcuts import render_to_response, render_to_string
 try:
     from django.views.decorators.csrf import csrf_exempt
@@ -27,84 +22,103 @@ except ImportError:
     from django.contrib.csrf.middleware import csrf_exempt
 from django_future.csrf import ensure_csrf_cookie
 from util.cache import cache_if_anonymous
-    
-from django_openid_auth import auth as openid_auth
-from openid.consumer.consumer import (Consumer, SUCCESS, CANCEL, FAILURE)
-import django_openid_auth.views as openid_views
 
-from openid.server.server import Server, ProtocolError, CheckIDRequest, EncodingError
+import django_openid_auth.views as openid_views
+from django_openid_auth import auth as openid_auth
+from openid.consumer.consumer import SUCCESS
+
+from openid.server.server import Server
 from openid.server.trustroot import TrustRoot
 from openid.store.filestore import FileOpenIDStore
-from openid.yadis.discover import DiscoveryFailure
-from openid.consumer.discover import OPENID_IDP_2_0_TYPE
 from openid.extensions import ax, sreg
-from openid.fetchers import HTTPFetchingError
 
 import student.views as student_views
 
 log = logging.getLogger("mitx.external_auth")
 
+
 @csrf_exempt
-def default_render_failure(request, message, status=403, template_name='extauth_failure.html', exception=None):
+def default_render_failure(request,
+                           message,
+                           status=403,
+                           template_name='extauth_failure.html',
+                           exception=None):
     """Render an Openid error page to the user."""
     message = "In openid_failure " + message
     log.debug(message)
-    data = render_to_string( template_name, dict(message=message, exception=exception))
+    data = render_to_string(template_name,
+                            dict(message=message, exception=exception))
     return HttpResponse(data, status=status)
 
 #-----------------------------------------------------------------------------
 # Openid
 
+
 def edXauth_generate_password(length=12, chars=string.letters + string.digits):
     """Generate internal password for externally authenticated user"""
     return ''.join([random.choice(chars) for i in range(length)])
 
+
 @csrf_exempt
-def edXauth_openid_login_complete(request,  redirect_field_name=REDIRECT_FIELD_NAME, render_failure=None):
+def edXauth_openid_login_complete(request,
+                                  redirect_field_name=REDIRECT_FIELD_NAME,
+                                  render_failure=None):
     """Complete the openid login process"""
 
-    redirect_to = request.REQUEST.get(redirect_field_name, '')
-    render_failure = render_failure or \
-                     getattr(settings, 'OPENID_RENDER_FAILURE', None) or \
-                     default_render_failure
-                                                   
+    redirect_to = request.REQUEST.get(redirect_field_name, '')  # TODO: [rocha] redirect_to never used?
+
+    render_failure = (render_failure or
+                      getattr(settings, 'OPENID_RENDER_FAILURE', None) or
+                      default_render_failure)
+
     openid_response = openid_views.parse_openid_response(request)
     if not openid_response:
-        return render_failure(request, 'This is an OpenID relying party endpoint.')
+        return render_failure(request,
+                              'This is an OpenID relying party endpoint.')
 
     if openid_response.status == SUCCESS:
         external_id = openid_response.identity_url
-        oid_backend =  openid_auth.OpenIDBackend()
+        oid_backend = openid_auth.OpenIDBackend()
         details = oid_backend._extract_user_details(openid_response)
 
         log.debug('openid success, details=%s' % details)
 
+        external_domain = "openid:%s" % settings.OPENID_SSO_SERVER_URL
+        fullname = '%s %s' % (details.get('first_name', ''),
+                              details.get('last_name', ''))
+
         return edXauth_external_login_or_signup(request,
-                                                external_id, 
-                                                "openid:%s" % settings.OPENID_SSO_SERVER_URL,
+                                                external_id,
+                                                external_domain,
                                                 details,
-                                                details.get('email',''),
-                                                '%s %s' % (details.get('first_name',''),details.get('last_name',''))
+                                                details.get('email', ''),
+                                                fullname,
                                                 )
-                                 
+
     return render_failure(request, 'Openid failure')
 
+
 #-----------------------------------------------------------------------------
 # generic external auth login or signup
 
-def edXauth_external_login_or_signup(request, external_id, external_domain, credentials, email, fullname,
+def edXauth_external_login_or_signup(request,
+                                     external_id,
+                                     external_domain,
+                                     credentials,
+                                     email,
+                                     fullname,
                                      retfun=None):
     # see if we have a map from this external_id to an edX username
     try:
-        eamap = ExternalAuthMap.objects.get(external_id = external_id,
-                                            external_domain = external_domain,
+        eamap = ExternalAuthMap.objects.get(external_id=external_id,
+                                            external_domain=external_domain,
                                             )
         log.debug('Found eamap=%s' % eamap)
     except ExternalAuthMap.DoesNotExist:
         # go render form for creating edX user
-        eamap = ExternalAuthMap(external_id = external_id,
-                                external_domain = external_domain,
-                                external_credentials = json.dumps(credentials),
+        eamap = ExternalAuthMap(external_id=external_id,
+                                external_domain=external_domain,
+                                external_credentials=json.dumps(credentials),
                                 )
         eamap.external_email = email
         eamap.external_name = fullname
@@ -115,20 +129,23 @@ def edXauth_external_login_or_signup(request, external_id, external_domain, cred
 
     internal_user = eamap.user
     if internal_user is None:
-        log.debug('ExtAuth: no user for %s yet, doing signup' % eamap.external_email)
+        log.debug('ExtAuth: no user for %s yet, doing signup' %
+                  eamap.external_email)
         return edXauth_signup(request, eamap)
-    
+
     uname = internal_user.username
     user = authenticate(username=uname, password=eamap.internal_password)
     if user is None:
-        log.warning("External Auth Login failed for %s / %s" % (uname,eamap.internal_password))
+        log.warning("External Auth Login failed for %s / %s" %
+                    (uname, eamap.internal_password))
         return edXauth_signup(request, eamap)
 
     if not user.is_active:
         log.warning("External Auth: user %s is not active" % (uname))
         # TODO: improve error page
-        return render_failure(request, 'Account not yet activated: please look for link in your email')
-     
+        msg = 'Account not yet activated: please look for link in your email'
+        return render_failure(request, msg)  # TODO: [rocha] render_failure not defined?
+
     login(request, user)
     request.session.set_expiry(0)
     student_views.try_change_enrollment(request)
@@ -136,8 +153,8 @@ def edXauth_external_login_or_signup(request, external_id, external_domain, cred
     if retfun is None:
         return redirect('/')
     return retfun()
-        
-    
+
+
 #-----------------------------------------------------------------------------
 # generic external auth signup
 
@@ -153,31 +170,36 @@ def edXauth_signup(request, eamap=None):
     eamap is an ExteralAuthMap object, specifying the external user
     for which to complete the signup.
     """
-    
+
     if eamap is None:
         pass
 
-    request.session['ExternalAuthMap'] = eamap	# save this for use by student.views.create_account
-    
+    # save this for use by student.views.create_account
+    request.session['ExternalAuthMap'] = eamap
+
+    # default conjoin name, no spaces
+    username = eamap.external_name.replace(' ', '')
+
     context = {'has_extauth_info': True,
-               'show_signup_immediately' : True,
+               'show_signup_immediately': True,
                'extauth_email': eamap.external_email,
-               'extauth_username' : eamap.external_name.replace(' ',''), # default - conjoin name, no spaces
+               'extauth_username': username,
                'extauth_name': eamap.external_name,
                }
-    
+
     log.debug('ExtAuth: doing signup for %s' % eamap.external_email)
 
     return student_views.index(request, extra_context=context)
 
+
 #-----------------------------------------------------------------------------
 # MIT SSL
 
 def ssl_dn_extract_info(dn):
     '''
-    Extract username, email address (may be anyuser@anydomain.com) and full name
-    from the SSL DN string.  Return (user,email,fullname) if successful, and None
-    otherwise.
+    Extract username, email address (may be anyuser@anydomain.com) and
+    full name from the SSL DN string.  Return (user,email,fullname) if
+    successful, and None otherwise.
     '''
     ss = re.search('/emailAddress=(.*)@([^/]+)', dn)
     if ss:
@@ -192,43 +214,50 @@ def ssl_dn_extract_info(dn):
         return None
     return (user, email, fullname)
 
+
 @csrf_exempt
 def edXauth_ssl_login(request):
     """
-    This is called by student.views.index when MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True
+    This is called by student.views.index when
+    MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True
 
-    Used for MIT user authentication.  This presumes the web server (nginx) has been configured 
-    to require specific client certificates.
+    Used for MIT user authentication.  This presumes the web server
+    (nginx) has been configured to require specific client
+    certificates.
 
-    If the incoming protocol is HTTPS (SSL) then authenticate via client certificate.  
-    The certificate provides user email and fullname; this populates the ExternalAuthMap.
-    The user is nevertheless still asked to complete the edX signup.  
+    If the incoming protocol is HTTPS (SSL) then authenticate via
+    client certificate.  The certificate provides user email and
+    fullname; this populates the ExternalAuthMap.  The user is
+    nevertheless still asked to complete the edX signup.
 
     Else continues on with student.views.index, and no authentication.
     """
-    certkey = "SSL_CLIENT_S_DN"	 # specify the request.META field to use
-    
-    cert = request.META.get(certkey,'')
+    certkey = "SSL_CLIENT_S_DN"  # specify the request.META field to use
+
+    cert = request.META.get(certkey, '')
     if not cert:
-        cert = request.META.get('HTTP_'+certkey,'')
+        cert = request.META.get('HTTP_' + certkey, '')
     if not cert:
         try:
-            cert = request._req.subprocess_env.get(certkey,'')	 # try the direct apache2 SSL key
-        except Exception as err:
+            # try the direct apache2 SSL key
+            cert = request._req.subprocess_env.get(certkey, '')
+        except Exception:
             pass
     if not cert:
         # no certificate information - go onward to main index
         return student_views.index(request)
 
     (user, email, fullname) = ssl_dn_extract_info(cert)
-    
-    return edXauth_external_login_or_signup(request, 
-                                            external_id=email, 
-                                            external_domain="ssl:MIT", 
-                                            credentials=cert, 
+
+    retfun = functools.partial(student_views.index, request)
+    return edXauth_external_login_or_signup(request,
+                                            external_id=email,
+                                            external_domain="ssl:MIT",
+                                            credentials=cert,
                                             email=email,
                                             fullname=fullname,
-                                            retfun = functools.partial(student_views.index, request))
+                                            retfun=retfun)
+
 
 def get_dict_for_openid(data):
     """
@@ -237,6 +266,7 @@ def get_dict_for_openid(data):
 
     return dict((k, v) for k, v in data.iteritems())
 
+
 def get_xrds_url(resource, request):
     """
     Return the XRDS url for a resource
@@ -250,12 +280,13 @@ def get_xrds_url(resource, request):
 
     return url
 
+
 def provider_respond(server, request, response, data):
     """
     Respond to an OpenID request
     """
 
-    # get simple registration request 
+    # get simple registration request
     sreg_data = {}
     sreg_request = sreg.SRegRequest.fromOpenIDRequest(request)
     sreg_fields = sreg_request.allRequestedFields()
@@ -269,7 +300,8 @@ def provider_respond(server, request, response, data):
                 sreg_data['fullname'] = data['fullname']
 
         # construct sreg response
-        sreg_response = sreg.SRegResponse.extractResponse(sreg_request, sreg_data)
+        sreg_response = sreg.SRegResponse.extractResponse(sreg_request,
+                                                          sreg_data)
         sreg_response.toMessage(response.fields)
 
     # get attribute exchange request
@@ -287,9 +319,8 @@ def provider_respond(server, request, response, data):
             for type_uri in ax_request.requested_attributes.iterkeys():
                 if type_uri == 'http://axschema.org/contact/email' and 'email' in data:
                     ax_response.addValue('http://axschema.org/contact/email', data['email'])
-
                 elif type_uri == 'http://axschema.org/namePerson' and 'fullname' in data:
-                    ax_response.addValue('http://axschema.org/namePerson', data['fullname']);
+                    ax_response.addValue('http://axschema.org/namePerson', data['fullname'])
 
             # construct ax response
             ax_response.toMessage(response.fields)
@@ -313,24 +344,24 @@ def validate_trust_root(openid_request):
 
     # verify the trust root/return to
     trust_root = openid_request.trust_root
-    return_to = openid_request.return_to
+    return_to = openid_request.return_to  # TODO: [rocha] never used?
 
     # don't allow empty trust roots
     if openid_request.trust_root is None:
-        return false 
+        return False
 
     # ensure trust root parses cleanly (one wildcard, of form *.foo.com, etc.)
-    trust_root = TrustRoot.parse(openid_request.trust_root) 
+    trust_root = TrustRoot.parse(openid_request.trust_root)
     if trust_root is None:
-        return false
+        return False
 
     # don't allow empty return tos
     if openid_request.return_to is None:
-        return false
+        return False
 
     # ensure return to is within trust root
     if not trust_root.validateURL(openid_request.return_to):
-        return false
+        return False
 
     # only allow *.cs50.net for now
     return trust_root.host.endswith('cs50.net')
@@ -356,11 +387,12 @@ def provider_login(request):
 
         # don't allow invalid and non-*.cs50.net trust roots
         if not validate_trust_root(openid_request):
-            return default_render_failure(request, "Invalid OpenID trust root") 
+            return default_render_failure(request, "Invalid OpenID trust root")
 
         # checkid_immediate not supported, require user interaction
         if openid_request.mode == 'checkid_immediate':
-            return provider_respond(server, openid_request, openid_request.answer(false), {})
+            return provider_respond(server, openid_request,
+                                    openid_request.answer(False), {})
 
         # checkid_setup, so display login page
         elif openid_request.mode == 'checkid_setup':
@@ -378,7 +410,8 @@ def provider_login(request):
 
         # OpenID response
         else:
-            return provider_respond(server, openid_request, server.handleRequest(openid_request), {})
+            return provider_respond(server, openid_request,
+                                    server.handleRequest(openid_request), {})
 
     # handle login
     if request.method == 'POST' and 'openid_request' in request.session:
@@ -388,7 +421,7 @@ def provider_login(request):
 
         # don't allow invalid and non-*.cs50.net trust roots
         if not validate_trust_root(openid_request):
-            return default_render_failure(request, "Invalid OpenID trust root") 
+            return default_render_failure(request, "Invalid OpenID trust root")
 
         # check if user with given email exists
         email = request.POST['email']
@@ -397,7 +430,8 @@ def provider_login(request):
             user = User.objects.get(email=email)
         except User.DoesNotExist:
             request.session['openid_error'] = True
-            log.warning("OpenID login failed - Unknown user email: {0}".format(email))
+            msg = "OpenID login failed - Unknown user email: {0}".format(email)
+            log.warning(msg)
             return HttpResponseRedirect(openid_request['url'])
 
         # attempt to authenticate user
@@ -405,7 +439,8 @@ def provider_login(request):
         user = authenticate(username=username, password=password)
         if user is None:
             request.session['openid_error'] = True
-            log.warning("OpenID login failed - password for {0} is invalid".format(email))
+            msg = "OpenID login failed - password for {0} is invalid".format(email)
+            log.warning(msg)
             return HttpResponseRedirect(openid_request['url'])
 
         # authentication succeeded, so log user in
@@ -416,14 +451,19 @@ def provider_login(request):
 
             # fullname field comes from user profile
             profile = UserProfile.objects.get(user=user)
-            log.info("OpenID login success - {0} ({1})".format(user.username, user.email))
+            log.info("OpenID login success - {0} ({1})".format(user.username,
+                                                               user.email))
 
             # redirect user to return_to location
             response = openid_request['request'].answer(True, None, endpoint + urlquote(user.username))
-            return provider_respond(server, openid_request['request'], response, {
-                'fullname': profile.name,
-                'email': user.email
-            })
+
+            return provider_respond(server,
+                                    openid_request['request'],
+                                    response,
+                                    {
+                                        'fullname': profile.name,
+                                        'email': user.email
+                                    })
 
         request.session['openid_error'] = True
         log.warning("Login failed - Account not active for user {0}".format(username))
@@ -445,6 +485,7 @@ def provider_login(request):
     response['X-XRDS-Location'] = get_xrds_url('xrds', request)
     return response
 
+
 def provider_identity(request):
     """
     XRDS for identity discovery
@@ -458,6 +499,7 @@ def provider_identity(request):
     response['X-XRDS-Location'] = get_xrds_url('identity', request)
     return response
 
+
 def provider_xrds(request):
     """
     XRDS for endpoint discovery
@@ -470,4 +512,3 @@ def provider_xrds(request):
     # custom XRDS header necessary for discovery process
     response['X-XRDS-Location'] = get_xrds_url('xrds', request)
     return response
-

From f5754f6d1e39c00b94d4d18b78f5276ac6d67fef Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Wed, 29 Aug 2012 13:30:55 -0400
Subject: [PATCH 084/181] Add some rudimentary error checking to forum views

[Hopefully fix #34992235]
---
 .../django_comment_client/forum/views.py      | 203 ++++++++++--------
 lms/lib/comment_client/utils.py               |  22 +-
 2 files changed, 128 insertions(+), 97 deletions(-)

diff --git a/lms/djangoapps/django_comment_client/forum/views.py b/lms/djangoapps/django_comment_client/forum/views.py
index 62055f0ab7..ef6f70954e 100644
--- a/lms/djangoapps/django_comment_client/forum/views.py
+++ b/lms/djangoapps/django_comment_client/forum/views.py
@@ -1,6 +1,6 @@
 from django.contrib.auth.decorators import login_required
 from django.views.decorators.http import require_POST
-from django.http import HttpResponse
+from django.http import HttpResponse, Http404
 from django.utils import simplejson
 from django.core.context_processors import csrf
 from django.core.urlresolvers import reverse
@@ -30,7 +30,7 @@ def _general_discussion_id(course_id):
 def _should_perform_search(request):
     return bool(request.GET.get('text', False) or \
             request.GET.get('tags', False))
-        
+
 
 def render_accordion(request, course, discussion_id):
 
@@ -59,7 +59,7 @@ def render_discussion(request, course_id, threads, *args, **kwargs):
     }[discussion_type]
 
     base_url = {
-        'inline': (lambda: reverse('django_comment_client.forum.views.inline_discussion', args=[course_id, discussion_id])), 
+        'inline': (lambda: reverse('django_comment_client.forum.views.inline_discussion', args=[course_id, discussion_id])),
         'forum': (lambda: reverse('django_comment_client.forum.views.forum_form_discussion', args=[course_id])),
         'user': (lambda: reverse('django_comment_client.forum.views.user_profile', args=[course_id, user_id])),
     }[discussion_type]()
@@ -98,6 +98,10 @@ def render_user_discussion(*args, **kwargs):
     return render_discussion(discussion_type='user', *args, **kwargs)
 
 def get_threads(request, course_id, discussion_id=None):
+    """
+    This may raise cc.utils.CommentClientError or
+    cc.utils.CommentClientUnknownError if something goes wrong.
+    """
 
     default_query_params = {
         'page': 1,
@@ -122,10 +126,17 @@ def get_threads(request, course_id, discussion_id=None):
 
 # discussion per page is fixed for now
 def inline_discussion(request, course_id, discussion_id):
-    threads, query_params = get_threads(request, course_id, discussion_id)
+    try:
+        threads, query_params = get_threads(request, course_id, discussion_id)
+    except (cc.utils.CommentClientError, cc.utils.CommentClientUnknownError) as err:
+        # TODO (vshnayder): since none of this code seems to be aware of the fact that
+        # sometimes things go wrong, I suspect that the js client is also not
+        # checking for errors on request.  Check and fix as needed.
+        raise Http404
+
     html = render_inline_discussion(request, course_id, threads, discussion_id=discussion_id,  \
                                                                  query_params=query_params)
-    
+
     return utils.JsonResponse({
         'html': html,
         'discussion_data': map(utils.safe_content, threads),
@@ -143,37 +154,41 @@ def render_search_bar(request, course_id, discussion_id=None, text=''):
 
 def forum_form_discussion(request, course_id):
     course = get_course_with_access(request.user, course_id, 'load')
-    threads, query_params = get_threads(request, course_id)
-    content = render_forum_discussion(request, course_id, threads, discussion_id=_general_discussion_id(course_id), query_params=query_params)
+    try:
+        threads, query_params = get_threads(request, course_id)
+        content = render_forum_discussion(request, course_id, threads, discussion_id=_general_discussion_id(course_id), query_params=query_params)
 
-    if request.is_ajax():
-        return utils.JsonResponse({
-            'html': content,
-            'discussion_data': map(utils.safe_content, threads),
-        })
-    else:
-        recent_active_threads = cc.search_recent_active_threads(
-            course_id,
-            recursive=False,
-            query_params={'follower_id': request.user.id},
-        )
+        if request.is_ajax():
+            return utils.JsonResponse({
+                'html': content,
+                'discussion_data': map(utils.safe_content, threads),
+            })
+        else:
+            recent_active_threads = cc.search_recent_active_threads(
+                course_id,
+                recursive=False,
+                query_params={'follower_id': request.user.id},
+            )
+
+            trending_tags = cc.search_trending_tags(
+                course_id,
+            )
+            context = {
+                'csrf': csrf(request)['csrf_token'],
+                'course': course,
+                'content': content,
+                'recent_active_threads': recent_active_threads,
+                'trending_tags': trending_tags,
+                'staff_access' : has_access(request.user, course, 'staff'),
+            }
+            # print "start rendering.."
+            return render_to_response('discussion/index.html', context)
+    except (cc.utils.CommentClientError, cc.utils.CommentClientUnknownError) as err:
+        raise Http404
 
-        trending_tags = cc.search_trending_tags(
-            course_id,
-        )
-        context = {
-            'csrf': csrf(request)['csrf_token'],
-            'course': course,
-            'content': content,
-            'recent_active_threads': recent_active_threads,
-            'trending_tags': trending_tags,
-            'staff_access' : has_access(request.user, course, 'staff'),
-        }
-        # print "start rendering.."
-        return render_to_response('discussion/index.html', context)
 
 def render_single_thread(request, discussion_id, course_id, thread_id):
-    
+
     thread = cc.Thread.find(thread_id).retrieve(recursive=True).to_dict()
 
     user_info = cc.User.from_django_user(request.user).to_dict()
@@ -192,75 +207,81 @@ def render_single_thread(request, discussion_id, course_id, thread_id):
 
 def single_thread(request, course_id, discussion_id, thread_id):
 
-    if request.is_ajax():
-        
-        user_info = cc.User.from_django_user(request.user).to_dict()
-        thread = cc.Thread.find(thread_id).retrieve(recursive=True)
-        annotated_content_info = utils.get_annotated_content_infos(course_id, thread, request.user, user_info=user_info)
-        context = {'thread': thread.to_dict(), 'course_id': course_id}
-        html = render_to_string('discussion/_ajax_single_thread.html', context)
+    try:
+        if request.is_ajax():
 
-        return utils.JsonResponse({
-            'html': html,
-            'content': utils.safe_content(thread.to_dict()),
-            'annotated_content_info': annotated_content_info,
-        })
+            user_info = cc.User.from_django_user(request.user).to_dict()
+            thread = cc.Thread.find(thread_id).retrieve(recursive=True)
+            annotated_content_info = utils.get_annotated_content_infos(course_id, thread, request.user, user_info=user_info)
+            context = {'thread': thread.to_dict(), 'course_id': course_id}
+            html = render_to_string('discussion/_ajax_single_thread.html', context)
 
-    else:
-        course = get_course_with_access(request.user, course_id, 'load')
+            return utils.JsonResponse({
+                'html': html,
+                'content': utils.safe_content(thread.to_dict()),
+                'annotated_content_info': annotated_content_info,
+            })
 
-        recent_active_threads = cc.search_recent_active_threads(
-            course_id,
-            recursive=False,
-            query_params={'follower_id': request.user.id},
-        )
+        else:
+            course = get_course_with_access(request.user, course_id, 'load')
 
-        trending_tags = cc.search_trending_tags(
-            course_id,
-        )
+            recent_active_threads = cc.search_recent_active_threads(
+                course_id,
+                recursive=False,
+                query_params={'follower_id': request.user.id},
+            )
 
-        context = {
-            'discussion_id': discussion_id,
-            'csrf': csrf(request)['csrf_token'],
-            'init': '',
-            'content': render_single_thread(request, discussion_id, course_id, thread_id),
-            'course': course,
-            'recent_active_threads': recent_active_threads,
-            'trending_tags': trending_tags,
-            'course_id': course.id,
-        }
+            trending_tags = cc.search_trending_tags(
+                course_id,
+            )
 
-        return render_to_response('discussion/single_thread.html', context)
+            context = {
+                'discussion_id': discussion_id,
+                'csrf': csrf(request)['csrf_token'],
+                'init': '',
+                'content': render_single_thread(request, discussion_id, course_id, thread_id),
+                'course': course,
+                'recent_active_threads': recent_active_threads,
+                'trending_tags': trending_tags,
+                'course_id': course.id,
+            }
+
+            return render_to_response('discussion/single_thread.html', context)
+    except (cc.utils.CommentClientError, cc.utils.CommentClientUnknownError) as err:
+        raise Http404
 
 def user_profile(request, course_id, user_id):
 
     course = get_course_with_access(request.user, course_id, 'load')
-    profiled_user = cc.User(id=user_id, course_id=course_id)
+    try:
+        profiled_user = cc.User(id=user_id, course_id=course_id)
 
-    query_params = {
-        'page': request.GET.get('page', 1),
-        'per_page': THREADS_PER_PAGE, # more than threads_per_page to show more activities
-    }
-
-    threads, page, num_pages = profiled_user.active_threads(query_params)
-
-    query_params['page'] = page
-    query_params['num_pages'] = num_pages
-
-    content = render_user_discussion(request, course_id, threads, user_id=user_id, query_params=query_params)
-
-    if request.is_ajax():
-        return utils.JsonResponse({
-            'html': content,
-            'discussion_data': map(utils.safe_content, threads),
-        })
-    else:
-        context = {
-            'course': course, 
-            'user': request.user,
-            'django_user': User.objects.get(id=user_id),
-            'profiled_user': profiled_user.to_dict(),
-            'content': content,
+        query_params = {
+            'page': request.GET.get('page', 1),
+            'per_page': THREADS_PER_PAGE, # more than threads_per_page to show more activities
         }
 
-        return render_to_response('discussion/user_profile.html', context)
+        threads, page, num_pages = profiled_user.active_threads(query_params)
+
+        query_params['page'] = page
+        query_params['num_pages'] = num_pages
+
+        content = render_user_discussion(request, course_id, threads, user_id=user_id, query_params=query_params)
+
+        if request.is_ajax():
+            return utils.JsonResponse({
+                'html': content,
+                'discussion_data': map(utils.safe_content, threads),
+            })
+        else:
+            context = {
+                'course': course,
+                'user': request.user,
+                'django_user': User.objects.get(id=user_id),
+                'profiled_user': profiled_user.to_dict(),
+                'content': content,
+            }
+
+            return render_to_response('discussion/user_profile.html', context)
+    except (cc.utils.CommentClientError, cc.utils.CommentClientUnknownError) as err:
+        raise Http404
diff --git a/lms/lib/comment_client/utils.py b/lms/lib/comment_client/utils.py
index 77e411424a..417d4eda8c 100644
--- a/lms/lib/comment_client/utils.py
+++ b/lms/lib/comment_client/utils.py
@@ -1,7 +1,10 @@
-import requests
 import json
+import logging
+import requests
 import settings
 
+log = logging.getLogger('mitx.' + __name__)
+
 def strip_none(dic):
     return dict([(k, v) for k, v in dic.iteritems() if v is not None])
 
@@ -18,15 +21,22 @@ def extract(dic, keys):
 
 def merge_dict(dic1, dic2):
     return dict(dic1.items() + dic2.items())
- 
+
 def perform_request(method, url, data_or_params=None, *args, **kwargs):
     if data_or_params is None:
         data_or_params = {}
     data_or_params['api_key'] = settings.API_KEY
-    if method in ['post', 'put', 'patch']:
-        response = requests.request(method, url, data=data_or_params)
-    else:
-        response = requests.request(method, url, params=data_or_params)
+    try:
+        if method in ['post', 'put', 'patch']:
+            response = requests.request(method, url, data=data_or_params)
+        else:
+            response = requests.request(method, url, params=data_or_params)
+    except Exception as err:
+        log.exception("Trying to call {method} on {url} with params {params}".format(
+            method=method, url=url, params=data_or_params))
+        # Reraise with a single exception type 
+        raise CommentClientError(str(err))
+
     if 200 < response.status_code < 500:
         raise CommentClientError(response.text)
     elif response.status_code == 500:

From c97db851fb230f3b602d5cfcea398eb4b53aa8be Mon Sep 17 00:00:00 2001
From: Nate Hardison <natehardison@gmail.com>
Date: Wed, 29 Aug 2012 15:28:44 -0400
Subject: [PATCH 085/181] Getting rid of unneeded variables

---
 common/djangoapps/external_auth/views.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py
index 149a185ee7..93bccbdc68 100644
--- a/common/djangoapps/external_auth/views.py
+++ b/common/djangoapps/external_auth/views.py
@@ -342,10 +342,6 @@ def validate_trust_root(openid_request):
     Only allow OpenID requests from valid trust roots
     """
 
-    # verify the trust root/return to
-    trust_root = openid_request.trust_root
-    return_to = openid_request.return_to  # TODO: [rocha] never used?
-
     # don't allow empty trust roots
     if openid_request.trust_root is None:
         return False

From 5c5d11409a3db07f14ed2a664920221806e4c16f Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Wed, 29 Aug 2012 13:58:22 -0700
Subject: [PATCH 086/181] Temporary fix to make uncaptioned videos display
 reasonably. Will replace with proper solution.

---
 common/lib/xmodule/xmodule/js/src/video/display.coffee      | 1 +
 .../xmodule/js/src/video/display/video_caption.coffee       | 6 +++++-
 common/lib/xmodule/xmodule/video_module.py                  | 2 ++
 lms/templates/video.html                                    | 2 +-
 4 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/common/lib/xmodule/xmodule/js/src/video/display.coffee b/common/lib/xmodule/xmodule/js/src/video/display.coffee
index ecf9ee1a72..3880091661 100644
--- a/common/lib/xmodule/xmodule/js/src/video/display.coffee
+++ b/common/lib/xmodule/xmodule/js/src/video/display.coffee
@@ -3,6 +3,7 @@ class @Video
     @el = $(element).find('.video')
     @id = @el.attr('id').replace(/video_/, '')
     @caption_data_dir = @el.data('caption-data-dir')
+    @show_captions = @el.data('show-captions') == "true"
     window.player = null
     @el = $("#video_#{@id}")
     @parseVideos @el.data('streams')
diff --git a/common/lib/xmodule/xmodule/js/src/video/display/video_caption.coffee b/common/lib/xmodule/xmodule/js/src/video/display/video_caption.coffee
index 3094cc5119..f65debb1a2 100644
--- a/common/lib/xmodule/xmodule/js/src/video/display/video_caption.coffee
+++ b/common/lib/xmodule/xmodule/js/src/video/display/video_caption.coffee
@@ -13,8 +13,12 @@ class @VideoCaption extends Subview
     "/static/#{@captionDataDir}/subs/#{@youtubeId}.srt.sjson"
 
   render: ->
+    # TODO: make it so you can have a video with no captions.
+    #@$('.video-wrapper').after """
+    #  <ol class="subtitles"><li>Attempting to load captions...</li></ol>
+    #  """
     @$('.video-wrapper').after """
-      <ol class="subtitles"><li>Attempting to load captions...</li></ol>
+      <ol class="subtitles"></ol>
       """
     @$('.video-controls .secondary-controls').append """
       <a href="#" class="hide-subtitles" title="Turn off captions">Captions</a>
diff --git a/common/lib/xmodule/xmodule/video_module.py b/common/lib/xmodule/xmodule/video_module.py
index 7d11dea283..a18a470272 100644
--- a/common/lib/xmodule/xmodule/video_module.py
+++ b/common/lib/xmodule/xmodule/video_module.py
@@ -30,6 +30,7 @@ class VideoModule(XModule):
         xmltree = etree.fromstring(self.definition['data'])
         self.youtube = xmltree.get('youtube')
         self.position = 0
+        self.show_captions = xmltree.get('show_captions', 'true')
 
         if instance_state is not None:
             state = json.loads(instance_state)
@@ -75,6 +76,7 @@ class VideoModule(XModule):
             'display_name': self.display_name,
             # TODO (cpennington): This won't work when we move to data that isn't on the filesystem
             'data_dir': self.metadata['data_dir'],
+            'show_captions': self.show_captions
         })
 
 
diff --git a/lms/templates/video.html b/lms/templates/video.html
index fcf25f3d69..bd3ec77fbe 100644
--- a/lms/templates/video.html
+++ b/lms/templates/video.html
@@ -2,7 +2,7 @@
   <h2> ${display_name} </h2>
 % endif
 
-<div id="video_${id}" class="video" data-streams="${streams}" data-caption-data-dir="${data_dir}">
+<div id="video_${id}" class="video" data-streams="${streams}" data-caption-data-dir="${data_dir}" data-show-captions="${show_captions}">
   <div class="tc-wrapper">
     <article class="video-wrapper">
       <section class="video-player">

From 1fd2f0c601a730ef1942a6e19e46d521ee7ec5a2 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Wed, 29 Aug 2012 18:02:37 -0400
Subject: [PATCH 087/181] Fix double show answer bug

---
 .../xmodule/js/src/capa/display.coffee        | 27 ++++++++++++-------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee
index ea5d939356..0ea6cffb58 100644
--- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee
+++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee
@@ -192,8 +192,11 @@ class @Problem
     if file_not_selected
       errors.push 'You did not select any files to submit'
 
-    if errors.length > 0
-        alert errors.join("\n")
+    error_html = '<ul>\n'
+    for error in errors
+      error_html += '<li>' + error + '</li>\n'
+    error_html += '</ul>'
+    @gentle_alert error_html
 
     abort_submission = file_too_large or file_not_selected or unallowed_file_submitted or required_files_not_submitted
 
@@ -208,7 +211,7 @@ class @Problem
             @render(response.contents)
             @updateProgress response
           else
-            alert(response.success)
+            @gentle_alert response.success
     
     if not abort_submission
       $.ajaxWithPrefix("#{@url}/problem_check", settings)
@@ -220,8 +223,10 @@ class @Problem
         when 'incorrect', 'correct'
           @render(response.contents)
           @updateProgress response
+          if @el.hasClass 'showed'
+            @el.removeClass 'showed'
         else
-          alert(response.success)
+          @gentle_alert response.success
 
   reset: =>
     Logger.log 'problem_reset', @answers
@@ -253,15 +258,19 @@ class @Problem
       @el.removeClass 'showed'
       @$('.show').val 'Show Answer'
 
+  gentle_alert: (msg) =>
+    if @el.find('.capa_alert').length
+      @el.find('.capa_alert').remove()
+    alert_elem = "<div class='capa_alert'>" + msg + "</div>"
+    @el.find('.action').after(alert_elem)
+    @el.find('.capa_alert').animate(opacity: 0, 500).animate(opacity: 1, 500)
+    
   save: =>
     Logger.log 'problem_save', @answers
     $.postWithPrefix "#{@url}/problem_save", @answers, (response) =>
       if response.success
-        if @el.find('.save_message').length
-            @el.find('.save_message').animate(opacity: 0, 500).animate(opacity: 1, 500)
-        else
-            saveMessage = "<div class='save_message'>Your answers have been saved but not graded. Hit 'Check' to grade them.</div>"
-            @el.find('.action').after(saveMessage)
+        saveMessage = "Your answers have been saved but not graded. Hit 'Check' to grade them."
+        @gentle_alert saveMessage
       @updateProgress response
 
   refreshMath: (event, element) =>

From cd4aab13fbe206eb1a13f75e8022eb4942888129 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Andr=C3=A9s=20Rocha?= <rocha@edx.org>
Date: Wed, 29 Aug 2012 18:40:12 -0400
Subject: [PATCH 088/181] [34078525] Added OPENID_PROVIDER_TRUSTED_ROOTS django
 setting

Added OPENID_PROVIDER_TRUSTED_ROOTS django setting. It should be a
list of glob matching patterns: ['*.cs50.net', '*.other.net']

Also did some minor refactoring and cleanup.
---
 common/djangoapps/external_auth/views.py | 159 ++++++++++++++---------
 lms/envs/common.py                       |   4 +
 lms/envs/dev.py                          |   4 +-
 3 files changed, 104 insertions(+), 63 deletions(-)

diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py
index 93bccbdc68..8b8443da68 100644
--- a/common/djangoapps/external_auth/views.py
+++ b/common/djangoapps/external_auth/views.py
@@ -4,6 +4,7 @@ import logging
 import random
 import re
 import string
+import fnmatch
 
 from external_auth.models import ExternalAuthMap
 
@@ -37,21 +38,30 @@ import student.views as student_views
 log = logging.getLogger("mitx.external_auth")
 
 
+# -----------------------------------------------------------------------------
+# OpenID Common
+# -----------------------------------------------------------------------------
+
+
 @csrf_exempt
 def default_render_failure(request,
                            message,
                            status=403,
                            template_name='extauth_failure.html',
                            exception=None):
-    """Render an Openid error page to the user."""
-    message = "In openid_failure " + message
-    log.debug(message)
+    """Render an Openid error page to the user"""
+
+    log.debug("In openid_failure " + message)
+
     data = render_to_string(template_name,
                             dict(message=message, exception=exception))
+
     return HttpResponse(data, status=status)
 
-#-----------------------------------------------------------------------------
-# Openid
+
+# -----------------------------------------------------------------------------
+# OpenID Authentication
+# -----------------------------------------------------------------------------
 
 
 def edXauth_generate_password(length=12, chars=string.letters + string.digits):
@@ -65,11 +75,7 @@ def edXauth_openid_login_complete(request,
                                   render_failure=None):
     """Complete the openid login process"""
 
-    redirect_to = request.REQUEST.get(redirect_field_name, '')  # TODO: [rocha] redirect_to never used?
-
-    render_failure = (render_failure or
-                      getattr(settings, 'OPENID_RENDER_FAILURE', None) or
-                      default_render_failure)
+    render_failure = (render_failure or default_render_failure)
 
     openid_response = openid_views.parse_openid_response(request)
     if not openid_response:
@@ -83,7 +89,8 @@ def edXauth_openid_login_complete(request,
 
         log.debug('openid success, details=%s' % details)
 
-        external_domain = "openid:%s" % settings.OPENID_SSO_SERVER_URL
+        url = getattr(settings, 'OPENID_SSO_SERVER_URL', None)
+        external_domain = "openid:%s" % url
         fullname = '%s %s' % (details.get('first_name', ''),
                               details.get('last_name', ''))
 
@@ -92,15 +99,11 @@ def edXauth_openid_login_complete(request,
                                                 external_domain,
                                                 details,
                                                 details.get('email', ''),
-                                                fullname,
-                                                )
+                                                fullname)
 
     return render_failure(request, 'Openid failure')
 
 
-#-----------------------------------------------------------------------------
-# generic external auth login or signup
-
 def edXauth_external_login_or_signup(request,
                                      external_id,
                                      external_domain,
@@ -108,29 +111,28 @@ def edXauth_external_login_or_signup(request,
                                      email,
                                      fullname,
                                      retfun=None):
+    """Generic external auth login or signup"""
+
     # see if we have a map from this external_id to an edX username
     try:
         eamap = ExternalAuthMap.objects.get(external_id=external_id,
-                                            external_domain=external_domain,
-                                            )
+                                            external_domain=external_domain)
         log.debug('Found eamap=%s' % eamap)
     except ExternalAuthMap.DoesNotExist:
         # go render form for creating edX user
         eamap = ExternalAuthMap(external_id=external_id,
                                 external_domain=external_domain,
-                                external_credentials=json.dumps(credentials),
-                                )
+                                external_credentials=json.dumps(credentials))
         eamap.external_email = email
         eamap.external_name = fullname
         eamap.internal_password = edXauth_generate_password()
-        log.debug('created eamap=%s' % eamap)
+        log.debug('Created eamap=%s' % eamap)
 
         eamap.save()
 
     internal_user = eamap.user
     if internal_user is None:
-        log.debug('ExtAuth: no user for %s yet, doing signup' %
-                  eamap.external_email)
+        log.debug('No user for %s yet, doing signup' % eamap.external_email)
         return edXauth_signup(request, eamap)
 
     uname = internal_user.username
@@ -141,10 +143,10 @@ def edXauth_external_login_or_signup(request,
         return edXauth_signup(request, eamap)
 
     if not user.is_active:
-        log.warning("External Auth: user %s is not active" % (uname))
+        log.warning("User %s is not active" % (uname))
         # TODO: improve error page
         msg = 'Account not yet activated: please look for link in your email'
-        return render_failure(request, msg)  # TODO: [rocha] render_failure not defined?
+        return default_render_failure(request, msg)
 
     login(request, user)
     request.session.set_expiry(0)
@@ -155,9 +157,6 @@ def edXauth_external_login_or_signup(request,
     return retfun()
 
 
-#-----------------------------------------------------------------------------
-# generic external auth signup
-
 @ensure_csrf_cookie
 @cache_if_anonymous
 def edXauth_signup(request, eamap=None):
@@ -187,20 +186,22 @@ def edXauth_signup(request, eamap=None):
                'extauth_name': eamap.external_name,
                }
 
-    log.debug('ExtAuth: doing signup for %s' % eamap.external_email)
+    log.debug('Doing signup for %s' % eamap.external_email)
 
     return student_views.index(request, extra_context=context)
 
 
-#-----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # MIT SSL
+# -----------------------------------------------------------------------------
+
 
 def ssl_dn_extract_info(dn):
-    '''
+    """
     Extract username, email address (may be anyuser@anydomain.com) and
     full name from the SSL DN string.  Return (user,email,fullname) if
     successful, and None otherwise.
-    '''
+    """
     ss = re.search('/emailAddress=(.*)@([^/]+)', dn)
     if ss:
         user = ss.group(1)
@@ -259,6 +260,11 @@ def edXauth_ssl_login(request):
                                             retfun=retfun)
 
 
+# -----------------------------------------------------------------------------
+# OpenID Provider
+# -----------------------------------------------------------------------------
+
+
 def get_dict_for_openid(data):
     """
     Return a dictionary suitable for the OpenID library
@@ -281,12 +287,7 @@ def get_xrds_url(resource, request):
     return url
 
 
-def provider_respond(server, request, response, data):
-    """
-    Respond to an OpenID request
-    """
-
-    # get simple registration request
+def add_openid_simple_registration(request, response, data):
     sreg_data = {}
     sreg_request = sreg.SRegRequest.fromOpenIDRequest(request)
     sreg_fields = sreg_request.allRequestedFields()
@@ -304,27 +305,38 @@ def provider_respond(server, request, response, data):
                                                           sreg_data)
         sreg_response.toMessage(response.fields)
 
-    # get attribute exchange request
+
+def add_openid_attribute_exchange(request, response, data):
     try:
         ax_request = ax.FetchRequest.fromOpenIDRequest(request)
-
     except ax.AXError:
+        #  not using OpenID attribute exchange extension
         pass
-
     else:
         ax_response = ax.FetchResponse()
 
         # if consumer requested attribute exchange fields, add them
         if ax_request and ax_request.requested_attributes:
             for type_uri in ax_request.requested_attributes.iterkeys():
-                if type_uri == 'http://axschema.org/contact/email' and 'email' in data:
-                    ax_response.addValue('http://axschema.org/contact/email', data['email'])
-                elif type_uri == 'http://axschema.org/namePerson' and 'fullname' in data:
-                    ax_response.addValue('http://axschema.org/namePerson', data['fullname'])
+                email_schema = 'http://axschema.org/contact/email'
+                name_schema = 'http://axschema.org/namePerson'
+                if type_uri == email_schema and 'email' in data:
+                    ax_response.addValue(email_schema, data['email'])
+                elif type_uri == name_schema and 'fullname' in data:
+                    ax_response.addValue(name_schema, data['fullname'])
 
             # construct ax response
             ax_response.toMessage(response.fields)
 
+
+def provider_respond(server, request, response, data):
+    """
+    Respond to an OpenID request
+    """
+    # get and add extensions
+    add_openid_simple_registration(request, response, data)
+    add_openid_attribute_exchange(request, response, data)
+
     # create http response from OpenID response
     webresponse = server.encodeResponse(response)
     http_response = HttpResponse(webresponse.body)
@@ -342,25 +354,44 @@ def validate_trust_root(openid_request):
     Only allow OpenID requests from valid trust roots
     """
 
+    trusted_roots = getattr(settings, 'OPENID_PROVIDER_TRUSTED_ROOT', None)
+
+    if trusted_roots is None:
+        log.debug('not using trusted roots')
+        # not using trusted roots
+        return True
+
+    log.debug('validating trusted roots')
+
     # don't allow empty trust roots
-    if openid_request.trust_root is None:
+    if (not hasattr(openid_request, 'trust_root') or
+            openid_request.trust_root is None):
+        log.debug('no trust_root')
         return False
 
     # ensure trust root parses cleanly (one wildcard, of form *.foo.com, etc.)
     trust_root = TrustRoot.parse(openid_request.trust_root)
     if trust_root is None:
+        log.debug('invalid trust_root')
         return False
 
     # don't allow empty return tos
-    if openid_request.return_to is None:
+    if (not hasattr(openid_request, 'return_to') or
+            openid_request.return_to is None):
+        log.debug('empty return_to')
         return False
 
     # ensure return to is within trust root
     if not trust_root.validateURL(openid_request.return_to):
+        log.debug('invalid return_to')
         return False
 
-    # only allow *.cs50.net for now
-    return trust_root.host.endswith('cs50.net')
+    # check that the root matches the ones we trust
+    if not any(r for r in trusted_roots if fnmatch.fnmatch(trust_root, r)):
+        log.debug('non-trusted root')
+        return False
+
+    return True
 
 
 @csrf_exempt
@@ -381,7 +412,7 @@ def provider_login(request):
         # decode request
         openid_request = server.decodeRequest(query)
 
-        # don't allow invalid and non-*.cs50.net trust roots
+        # don't allow invalid and non-trusted trust roots
         if not validate_trust_root(openid_request):
             return default_render_failure(request, "Invalid OpenID trust root")
 
@@ -420,8 +451,7 @@ def provider_login(request):
             return default_render_failure(request, "Invalid OpenID trust root")
 
         # check if user with given email exists
-        email = request.POST['email']
-        password = request.POST['password']
+        email = request.POST.get('email', None)
         try:
             user = User.objects.get(email=email)
         except User.DoesNotExist:
@@ -432,10 +462,12 @@ def provider_login(request):
 
         # attempt to authenticate user
         username = user.username
+        password = request.POST.get('password', None)
         user = authenticate(username=username, password=password)
         if user is None:
             request.session['openid_error'] = True
-            msg = "OpenID login failed - password for {0} is invalid".format(email)
+            msg = "OpenID login failed - password for {0} is invalid"
+            msg = msg.format(email)
             log.warning(msg)
             return HttpResponseRedirect(openid_request['url'])
 
@@ -451,7 +483,8 @@ def provider_login(request):
                                                                user.email))
 
             # redirect user to return_to location
-            response = openid_request['request'].answer(True, None, endpoint + urlquote(user.username))
+            url = endpoint + urlquote(user.username)
+            response = openid_request['request'].answer(True, None, url)
 
             return provider_respond(server,
                                     openid_request['request'],
@@ -462,13 +495,15 @@ def provider_login(request):
                                     })
 
         request.session['openid_error'] = True
-        log.warning("Login failed - Account not active for user {0}".format(username))
+        msg = "Login failed - Account not active for user {0}".format(username)
+        log.warning(msg)
         return HttpResponseRedirect(openid_request['url'])
 
     # determine consumer domain if applicable
     return_to = ''
     if 'openid.return_to' in request.REQUEST:
-        matches = re.match(r'\w+:\/\/([\w\.-]+)', request.REQUEST['openid.return_to'])
+        return_to = request.REQUEST['openid.return_to']
+        matches = re.match(r'\w+:\/\/([\w\.-]+)', return_to)
         return_to = matches.group(1)
 
     # display login page
@@ -487,9 +522,9 @@ def provider_identity(request):
     XRDS for identity discovery
     """
 
-    response = render_to_response('identity.xml', {
-        'url': get_xrds_url('login', request)
-    }, mimetype='text/xml')
+    response = render_to_response('identity.xml',
+                                  {'url': get_xrds_url('login', request)},
+                                  mimetype='text/xml')
 
     # custom XRDS header necessary for discovery process
     response['X-XRDS-Location'] = get_xrds_url('identity', request)
@@ -501,9 +536,9 @@ def provider_xrds(request):
     XRDS for endpoint discovery
     """
 
-    response = render_to_response('xrds.xml', {
-        'url': get_xrds_url('login', request)
-    }, mimetype='text/xml')
+    response = render_to_response('xrds.xml',
+                                  {'url': get_xrds_url('login', request)},
+                                  mimetype='text/xml')
 
     # custom XRDS header necessary for discovery process
     response['X-XRDS-Location'] = get_xrds_url('xrds', request)
diff --git a/lms/envs/common.py b/lms/envs/common.py
index 9825df338d..36a8d54d3c 100644
--- a/lms/envs/common.py
+++ b/lms/envs/common.py
@@ -120,6 +120,10 @@ node_paths = [COMMON_ROOT / "static/js/vendor",
               ]
 NODE_PATH = ':'.join(node_paths)
 
+
+############################ OpenID Provider  ##################################
+OPENID_PROVIDER_TRUSTED_ROOTS = ['cs50.net', '*.cs50.net'] 
+
 ################################## MITXWEB #####################################
 # This is where we stick our compiled template files. Most of the app uses Mako
 # templates
diff --git a/lms/envs/dev.py b/lms/envs/dev.py
index 1b5da7ff71..e1d6071bbf 100644
--- a/lms/envs/dev.py
+++ b/lms/envs/dev.py
@@ -17,7 +17,6 @@ MITX_FEATURES['DISABLE_START_DATES'] = True
 MITX_FEATURES['ENABLE_SQL_TRACKING_LOGS'] = True
 MITX_FEATURES['SUBDOMAIN_COURSE_LISTINGS'] = False  # Enable to test subdomains--otherwise, want all courses to show up
 MITX_FEATURES['SUBDOMAIN_BRANDING'] = True
-MITX_FEATURES['AUTH_USE_OPENID_PROVIDER'] = True
 
 WIKI_ENABLED = True
 
@@ -106,6 +105,7 @@ LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1']
 
 ################################ OpenID Auth #################################
 MITX_FEATURES['AUTH_USE_OPENID'] = True
+MITX_FEATURES['AUTH_USE_OPENID_PROVIDER'] = True
 MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True
 
 INSTALLED_APPS += ('external_auth',)
@@ -116,6 +116,8 @@ OPENID_UPDATE_DETAILS_FROM_SREG = True
 OPENID_SSO_SERVER_URL = 'https://www.google.com/accounts/o8/id'	# TODO: accept more endpoints
 OPENID_USE_AS_ADMIN_LOGIN = False
 
+OPENID_PROVIDER_TRUSTED_ROOTS = ['*'] 
+
 ################################ MIT Certificates SSL Auth #################################
 MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True
 

From 902a887e7917e31b9b4e98156f299140849219f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Andr=C3=A9s=20Rocha?= <rocha@edx.org>
Date: Wed, 29 Aug 2012 18:46:35 -0400
Subject: [PATCH 089/181] [34078525] pep8 renaming of some extern_auth
 functions

---
 common/djangoapps/external_auth/views.py | 58 ++++++++++++------------
 lms/djangoapps/branding/views.py         |  4 +-
 lms/urls.py                              |  2 +-
 3 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py
index 8b8443da68..4dacd99d28 100644
--- a/common/djangoapps/external_auth/views.py
+++ b/common/djangoapps/external_auth/views.py
@@ -64,15 +64,15 @@ def default_render_failure(request,
 # -----------------------------------------------------------------------------
 
 
-def edXauth_generate_password(length=12, chars=string.letters + string.digits):
+def generate_password(length=12, chars=string.letters + string.digits):
     """Generate internal password for externally authenticated user"""
     return ''.join([random.choice(chars) for i in range(length)])
 
 
 @csrf_exempt
-def edXauth_openid_login_complete(request,
-                                  redirect_field_name=REDIRECT_FIELD_NAME,
-                                  render_failure=None):
+def openid_login_complete(request,
+                          redirect_field_name=REDIRECT_FIELD_NAME,
+                          render_failure=None):
     """Complete the openid login process"""
 
     render_failure = (render_failure or default_render_failure)
@@ -94,23 +94,23 @@ def edXauth_openid_login_complete(request,
         fullname = '%s %s' % (details.get('first_name', ''),
                               details.get('last_name', ''))
 
-        return edXauth_external_login_or_signup(request,
-                                                external_id,
-                                                external_domain,
-                                                details,
-                                                details.get('email', ''),
-                                                fullname)
+        return external_login_or_signup(request,
+                                        external_id,
+                                        external_domain,
+                                        details,
+                                        details.get('email', ''),
+                                        fullname)
 
     return render_failure(request, 'Openid failure')
 
 
-def edXauth_external_login_or_signup(request,
-                                     external_id,
-                                     external_domain,
-                                     credentials,
-                                     email,
-                                     fullname,
-                                     retfun=None):
+def external_login_or_signup(request,
+                             external_id,
+                             external_domain,
+                             credentials,
+                             email,
+                             fullname,
+                             retfun=None):
     """Generic external auth login or signup"""
 
     # see if we have a map from this external_id to an edX username
@@ -125,7 +125,7 @@ def edXauth_external_login_or_signup(request,
                                 external_credentials=json.dumps(credentials))
         eamap.external_email = email
         eamap.external_name = fullname
-        eamap.internal_password = edXauth_generate_password()
+        eamap.internal_password = generate_password()
         log.debug('Created eamap=%s' % eamap)
 
         eamap.save()
@@ -133,14 +133,14 @@ def edXauth_external_login_or_signup(request,
     internal_user = eamap.user
     if internal_user is None:
         log.debug('No user for %s yet, doing signup' % eamap.external_email)
-        return edXauth_signup(request, eamap)
+        return signup(request, eamap)
 
     uname = internal_user.username
     user = authenticate(username=uname, password=eamap.internal_password)
     if user is None:
         log.warning("External Auth Login failed for %s / %s" %
                     (uname, eamap.internal_password))
-        return edXauth_signup(request, eamap)
+        return signup(request, eamap)
 
     if not user.is_active:
         log.warning("User %s is not active" % (uname))
@@ -159,7 +159,7 @@ def edXauth_external_login_or_signup(request,
 
 @ensure_csrf_cookie
 @cache_if_anonymous
-def edXauth_signup(request, eamap=None):
+def signup(request, eamap=None):
     """
     Present form to complete for signup via external authentication.
     Even though the user has external credentials, he/she still needs
@@ -217,7 +217,7 @@ def ssl_dn_extract_info(dn):
 
 
 @csrf_exempt
-def edXauth_ssl_login(request):
+def ssl_login(request):
     """
     This is called by student.views.index when
     MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True
@@ -251,13 +251,13 @@ def edXauth_ssl_login(request):
     (user, email, fullname) = ssl_dn_extract_info(cert)
 
     retfun = functools.partial(student_views.index, request)
-    return edXauth_external_login_or_signup(request,
-                                            external_id=email,
-                                            external_domain="ssl:MIT",
-                                            credentials=cert,
-                                            email=email,
-                                            fullname=fullname,
-                                            retfun=retfun)
+    return external_login_or_signup(request,
+                                    external_id=email,
+                                    external_domain="ssl:MIT",
+                                    credentials=cert,
+                                    email=email,
+                                    fullname=fullname,
+                                    retfun=retfun)
 
 
 # -----------------------------------------------------------------------------
diff --git a/lms/djangoapps/branding/views.py b/lms/djangoapps/branding/views.py
index e32eb92138..9fe912e947 100644
--- a/lms/djangoapps/branding/views.py
+++ b/lms/djangoapps/branding/views.py
@@ -20,8 +20,8 @@ def index(request):
         return redirect(reverse('dashboard'))
 
     if settings.MITX_FEATURES.get('AUTH_USE_MIT_CERTIFICATES'):
-        from external_auth.views import edXauth_ssl_login
-        return edXauth_ssl_login(request)
+        from external_auth.views import ssl_login
+        return ssl_login(request)
 
     university = branding.get_university(request.META.get('HTTP_HOST'))
     if university is None:
diff --git a/lms/urls.py b/lms/urls.py
index 380d168f62..d88aef17f6 100644
--- a/lms/urls.py
+++ b/lms/urls.py
@@ -215,7 +215,7 @@ if settings.DEBUG:
 if settings.MITX_FEATURES.get('AUTH_USE_OPENID'):
     urlpatterns += (
         url(r'^openid/login/$', 'django_openid_auth.views.login_begin', name='openid-login'),
-        url(r'^openid/complete/$', 'external_auth.views.edXauth_openid_login_complete', name='openid-complete'),
+        url(r'^openid/complete/$', 'external_auth.views.openid_login_complete', name='openid-complete'),
         url(r'^openid/logo.gif$', 'django_openid_auth.views.logo', name='openid-logo'),
     )
 

From 5d6bed863a22ee06305ed3f37dc62cef8c9b21f7 Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Wed, 29 Aug 2012 16:41:01 -0700
Subject: [PATCH 090/181] Fix discussions in xmodules

---
 common/lib/xmodule/xmodule/discussion_module.py             | 6 ++++++
 common/lib/xmodule/xmodule/js/src/discussion/display.coffee | 4 ++++
 lms/static/coffee/src/discussion/main.coffee                | 3 ---
 3 files changed, 10 insertions(+), 3 deletions(-)
 create mode 100644 common/lib/xmodule/xmodule/js/src/discussion/display.coffee

diff --git a/common/lib/xmodule/xmodule/discussion_module.py b/common/lib/xmodule/xmodule/discussion_module.py
index c029d95098..4bfaeee183 100644
--- a/common/lib/xmodule/xmodule/discussion_module.py
+++ b/common/lib/xmodule/xmodule/discussion_module.py
@@ -1,4 +1,5 @@
 from lxml import etree
+from pkg_resources import resource_string, resource_listdir
 
 from xmodule.x_module import XModule
 from xmodule.raw_module import RawDescriptor
@@ -6,6 +7,11 @@ from xmodule.raw_module import RawDescriptor
 import json
 
 class DiscussionModule(XModule):
+    js = {'coffee':
+            [resource_string(__name__, 'js/src/time.coffee'),
+            resource_string(__name__, 'js/src/discussion/display.coffee')]
+        }
+    js_module_name = "InlineDiscussion"
     def get_html(self):
         context = {
             'discussion_id': self.discussion_id,
diff --git a/common/lib/xmodule/xmodule/js/src/discussion/display.coffee b/common/lib/xmodule/xmodule/js/src/discussion/display.coffee
new file mode 100644
index 0000000000..27a67f5aab
--- /dev/null
+++ b/common/lib/xmodule/xmodule/js/src/discussion/display.coffee
@@ -0,0 +1,4 @@
+class @InlineDiscussion
+  constructor: (element) ->
+    @el = $(element).find('.discussion-module')
+    @view = new DiscussionModuleView(el: @el)
diff --git a/lms/static/coffee/src/discussion/main.coffee b/lms/static/coffee/src/discussion/main.coffee
index 023345c5da..2c6f4090cc 100644
--- a/lms/static/coffee/src/discussion/main.coffee
+++ b/lms/static/coffee/src/discussion/main.coffee
@@ -3,9 +3,6 @@ $ ->
   window.$$contents = {}
   window.$$discussions = {}
 
-  $(".discussion-module").each (index, elem) ->
-    view = new DiscussionModuleView(el: elem)
-
   $("section.discussion").each (index, elem) ->
     discussionData = DiscussionUtil.getDiscussionData($(elem).attr("_id"))
     discussion = new Discussion()

From e62f8d7562ada0f4d816757f23fe2ceb31ae73d7 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Wed, 29 Aug 2012 20:03:26 -0400
Subject: [PATCH 091/181] More info in StudentInputError

---
 common/lib/capa/capa/responsetypes.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index 273b1a3f8f..d1b58e53a9 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -8,6 +8,7 @@ Used by capa_problem.py
 '''
 
 # standard library imports
+import cgi
 import inspect
 import json
 import logging
@@ -725,7 +726,8 @@ class NumericalResponse(LoncapaResponse):
         # I think this is just pyparsing.ParseException, calc.UndefinedVariable:
         # But we'd need to confirm
         except:
-            raise StudentInputError('Invalid input -- please use a number only')
+            raise StudentInputError("Invalid input: could not parse '%s' as a number" %\
+                                    cgi.escape(student_answer))
 
         if correct:
             return CorrectMap(self.answer_id, 'correct')
@@ -1517,11 +1519,12 @@ class FormulaResponse(LoncapaResponse):
                                            cs=self.case_sensitive)
             except UndefinedVariable as uv:
                 log.debug('formularesponse: undefined variable in given=%s' % given)
-                raise StudentInputError(uv.message + " not permitted in answer")
+                raise StudentInputError("Invalid input: " + uv.message + " not permitted in answer")
             except Exception as err:
                 #traceback.print_exc()
                 log.debug('formularesponse: error %s in formula' % err)
-                raise StudentInputError("Error in formula")
+                raise StudentInputError("Invalid input: Could not parse '%s' as a formula" %\
+                                        cgi.escape(given))
             if numpy.isnan(student_result) or numpy.isinf(student_result):
                 return "incorrect"
             if not compare_with_tolerance(student_result, instructor_result, self.tolerance):

From bb55a76379ef0d139ff24f7fdc04faa4f231bef1 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Wed, 29 Aug 2012 22:01:16 -0400
Subject: [PATCH 092/181] Allow courseware to navigate to specific tabs of a
 sequence

---
 .../xmodule/js/src/sequence/display.coffee    | 30 +++++++++++++------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/common/lib/xmodule/xmodule/js/src/sequence/display.coffee b/common/lib/xmodule/xmodule/js/src/sequence/display.coffee
index 35a33a9ea1..ff6794ec10 100644
--- a/common/lib/xmodule/xmodule/js/src/sequence/display.coffee
+++ b/common/lib/xmodule/xmodule/js/src/sequence/display.coffee
@@ -2,6 +2,7 @@ class @Sequence
   constructor: (element) ->
     @el = $(element).find('.sequence')
     @contents = @$('.seq_contents')
+    @num_contents = @contents.length
     @id = @el.data('id')
     @modx_url = @el.data('course_modx_root')
     @initProgress()
@@ -90,18 +91,29 @@ class @Sequence
       @toggleArrows()
       @hookUpProgressEvent()
 
+      sequence_links = @$('#seq_content a.seqnav')
+      sequence_links.click @goto
+
   goto: (event) =>
     event.preventDefault()
-    new_position = $(event.target).data('element')
-    Logger.log "seq_goto", old: @position, new: new_position, id: @id
-    
-    # On Sequence chage, destroy any existing polling thread 
-    #   for queued submissions, see ../capa/display.coffee
-    if window.queuePollerID
-      window.clearTimeout(window.queuePollerID)
-      delete window.queuePollerID
+    # Links from within the coureware uses <a class='seqnav' href="n">...</a>
+    if $(event.target).hasClass 'seqnav'
+      new_position = $(event.target).attr('href')
+    else
+      new_position = $(event.target).data('element')
 
-    @render new_position
+    if (1 <= new_position) and (new_position <= @num_contents)
+      Logger.log "seq_goto", old: @position, new: new_position, id: @id
+    
+      # On Sequence chage, destroy any existing polling thread 
+      #   for queued submissions, see ../capa/display.coffee
+      if window.queuePollerID
+        window.clearTimeout(window.queuePollerID)
+        delete window.queuePollerID
+
+      @render new_position
+    else
+      alert 'Sequence error! Cannot navigate to tab ' + new_position + 'in the current SequenceModule. Please contact the course staff.'
 
   next: (event) =>
     event.preventDefault()

From cb07b0601e090370c17459e6a163e22b4363031d Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Wed, 29 Aug 2012 22:04:39 -0400
Subject: [PATCH 093/181] missing "settings." in line in lms_migration

---
 lms/djangoapps/lms_migration/migrate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lms/djangoapps/lms_migration/migrate.py b/lms/djangoapps/lms_migration/migrate.py
index a7d04a655d..0997682215 100644
--- a/lms/djangoapps/lms_migration/migrate.py
+++ b/lms/djangoapps/lms_migration/migrate.py
@@ -139,7 +139,7 @@ def gitreload(request, reload_dir=None):
 
     ALLOWED_IPS = []	# allow none by default
     if hasattr(settings,'ALLOWED_GITRELOAD_IPS'):	# allow override in settings
-        ALLOWED_IPS = ALLOWED_GITRELOAD_IPS
+        ALLOWED_IPS = settings.ALLOWED_GITRELOAD_IPS
 
     if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
         if request.user and request.user.is_staff:

From 86409a2a2644703826970ea462d04a78247c0df0 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Wed, 29 Aug 2012 22:11:26 -0400
Subject: [PATCH 094/181] Adjust comments for Sequence js

---
 common/lib/xmodule/xmodule/js/src/sequence/display.coffee | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/common/lib/xmodule/xmodule/js/src/sequence/display.coffee b/common/lib/xmodule/xmodule/js/src/sequence/display.coffee
index ff6794ec10..ab337d9b7e 100644
--- a/common/lib/xmodule/xmodule/js/src/sequence/display.coffee
+++ b/common/lib/xmodule/xmodule/js/src/sequence/display.coffee
@@ -96,10 +96,9 @@ class @Sequence
 
   goto: (event) =>
     event.preventDefault()
-    # Links from within the coureware uses <a class='seqnav' href="n">...</a>
-    if $(event.target).hasClass 'seqnav'
+    if $(event.target).hasClass 'seqnav' # Links from courseware <a class='seqnav' href='n'>...</a>
       new_position = $(event.target).attr('href')
-    else
+    else # Tab links generated by backend template
       new_position = $(event.target).data('element')
 
     if (1 <= new_position) and (new_position <= @num_contents)

From 7655e9eea05012800a034197796bd536625b45b5 Mon Sep 17 00:00:00 2001
From: Ibrahim Awwal <ibrahim.awwal@gmail.com>
Date: Tue, 28 Aug 2012 22:38:56 -0700
Subject: [PATCH 095/181] Ensure that js files are returned in alphabetical
 order by django-pipeline.

On most systems this seems to be the default behavior, but glob2.glob's order isn't guaranteed and on my machine the order seems to be random, which causes javascript which depends on include order to fail.
---
 lms/envs/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lms/envs/common.py b/lms/envs/common.py
index ce08bf9666..37d65e78f9 100644
--- a/lms/envs/common.py
+++ b/lms/envs/common.py
@@ -429,7 +429,7 @@ main_vendor_js = [
   'js/vendor/jquery.qtip.min.js',
 ]
 
-discussion_js = glob2.glob(PROJECT_ROOT / 'static/coffee/src/discussion/*.coffee')
+discussion_js = sorted(glob2.glob(PROJECT_ROOT / 'static/coffee/src/discussion/*.coffee'))
 
 # Load javascript from all of the available xmodules, and
 # prep it for use in pipeline js
@@ -497,10 +497,10 @@ PIPELINE_JS = {
         'source_filenames': [
             pth.replace(COMMON_ROOT / 'static/', '')
             for pth
-            in glob2.glob(COMMON_ROOT / 'static/coffee/src/**/*.coffee')
+            in sorted(glob2.glob(COMMON_ROOT / 'static/coffee/src/**/*.coffee'))
         ] + [
             pth.replace(PROJECT_ROOT / 'static/', '')
-            for pth in glob2.glob(PROJECT_ROOT / 'static/coffee/src/**/*.coffee')\
+            for pth in sorted(glob2.glob(PROJECT_ROOT / 'static/coffee/src/**/*.coffee'))\
             if pth not in courseware_only_js and pth not in discussion_js
         ] + [
             'js/form.ext.js',

From b13c5acd4fe03dcda0212e9bb1ba5751086b474c Mon Sep 17 00:00:00 2001
From: Ibrahim Awwal <ibrahim.awwal@gmail.com>
Date: Tue, 28 Aug 2012 22:42:11 -0700
Subject: [PATCH 096/181] Save the user's selected sort order on the comments
 service.

Requires cs_comments_service to have the field default_sort_key on users.
---
 lms/djangoapps/django_comment_client/forum/views.py | 13 ++++++++++++-
 lms/lib/comment_client/user.py                      |  4 ++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/lms/djangoapps/django_comment_client/forum/views.py b/lms/djangoapps/django_comment_client/forum/views.py
index 62055f0ab7..ae8e999a8a 100644
--- a/lms/djangoapps/django_comment_client/forum/views.py
+++ b/lms/djangoapps/django_comment_client/forum/views.py
@@ -102,7 +102,7 @@ def get_threads(request, course_id, discussion_id=None):
     default_query_params = {
         'page': 1,
         'per_page': THREADS_PER_PAGE,
-        'sort_key': 'activity',
+        'sort_key': 'date',
         'sort_order': 'desc',
         'text': '',
         'tags': '',
@@ -110,6 +110,17 @@ def get_threads(request, course_id, discussion_id=None):
         'course_id': course_id,
     }
 
+    if not request.GET.get('sort_key'):
+        # If the user did not select a sort key, use their last used sort key
+        user = cc.User.from_django_user(request.user)
+        user.retrieve()
+        default_query_params['sort_key'] = user.default_sort_key
+    else:
+        # If the user clicked a sort key, update their default sort key
+        user = cc.User.from_django_user(request.user)
+        user.default_sort_key = request.GET.get('sort_key')
+        user.save()
+
     query_params = merge_dict(default_query_params,
                               strip_none(extract(request.GET, ['page', 'sort_key', 'sort_order', 'text', 'tags'])))
 
diff --git a/lms/lib/comment_client/user.py b/lms/lib/comment_client/user.py
index ae4abf91b7..eea6fda407 100644
--- a/lms/lib/comment_client/user.py
+++ b/lms/lib/comment_client/user.py
@@ -8,10 +8,10 @@ class User(models.Model):
     accessible_fields = ['username', 'email', 'follower_ids', 'upvoted_ids', 'downvoted_ids',
                          'id', 'external_id', 'subscribed_user_ids', 'children', 'course_id',
                          'subscribed_thread_ids', 'subscribed_commentable_ids',
-                         'threads_count', 'comments_count',
+                         'threads_count', 'comments_count', 'default_sort_key'
                         ]
 
-    updatable_fields = ['username', 'external_id', 'email']
+    updatable_fields = ['username', 'external_id', 'email', 'default_sort_key']
     initializable_fields = updatable_fields
 
     base_url = "{prefix}/users".format(prefix=settings.PREFIX)

From cd84b63228a2954e22824ca243bb436cbca71238 Mon Sep 17 00:00:00 2001
From: Ibrahim Awwal <ibrahim.awwal@gmail.com>
Date: Wed, 29 Aug 2012 00:37:45 -0700
Subject: [PATCH 097/181] Make the search button line up with the search field
 better.

---
 lms/static/sass/_discussion.scss | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lms/static/sass/_discussion.scss b/lms/static/sass/_discussion.scss
index 6c1988684e..51f58730fe 100644
--- a/lms/static/sass/_discussion.scss
+++ b/lms/static/sass/_discussion.scss
@@ -501,7 +501,8 @@ $tag-text-color: #5b614f;
       font-size: inherit;
       font-weight: bold;
       margin-left: 1%;
-      padding-top: 9px;
+      padding-top: 4px;
+      padding-bottom: 2px;
       text-decoration: none;
     }
 

From 02e6fd17e4452b46f58713e0a0efb5cf340762db Mon Sep 17 00:00:00 2001
From: Ibrahim Awwal <ibrahim.awwal@gmail.com>
Date: Wed, 29 Aug 2012 22:20:20 -0700
Subject: [PATCH 098/181] Make 'date' the first sort key since it's the new
 default (students seem to prefer it, especially since top is kind of
 unclear).

---
 lms/templates/discussion/_sort.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lms/templates/discussion/_sort.html b/lms/templates/discussion/_sort.html
index 7ba10eab7b..99ba58f7c8 100644
--- a/lms/templates/discussion/_sort.html
+++ b/lms/templates/discussion/_sort.html
@@ -31,9 +31,9 @@
 
 <div class="discussion-sort local">
   <span class="discussion-label">Sort by:</span>
-  ${link_to_sort('activity', 'top')}
-  
   ${link_to_sort('date', 'date')}
+
+  ${link_to_sort('activity', 'top')}
   
   ${link_to_sort('votes', 'votes')}
   

From 4ce41c0111a122e0b7a91f492633c6924a5eab2a Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Thu, 30 Aug 2012 10:53:38 -0400
Subject: [PATCH 099/181] fix course_ids for Berkeley internal courses and
 3.091

---
 lms/envs/dev.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lms/envs/dev.py b/lms/envs/dev.py
index 97803cb47d..5da84f59f0 100644
--- a/lms/envs/dev.py
+++ b/lms/envs/dev.py
@@ -78,10 +78,10 @@ COURSE_LISTINGS = {
                 'MITx/3.091x/2012_Fall',
                 'MITx/6.002x/2012_Fall',
                 'MITx/6.00x/2012_Fall'],
-    'berkeley': ['BerkeleyX/CS169.1x/Cal_2012_Fall',
-                 'BerkeleyX/CS188.1x/Cal_2012_Fall'],
+    'berkeley': ['BerkeleyX/CS169/fa12',
+                 'BerkeleyX/CS188/fa12'],
     'harvard': ['HarvardX/CS50x/2012H'],
-    'mit': [],
+    'mit': ['MITx/3.091/MIT_2012_Fall'],
     'sjsu': ['MITx/6.002x-EE98/2012_Fall_SJSU'],
 }
 

From 87967b9eb8e75f66c4fa639e0f7f2d4b8ad062fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Andr=C3=A9s=20Rocha?= <rocha@edx.org>
Date: Thu, 30 Aug 2012 10:47:32 -0400
Subject: [PATCH 100/181] [34078525] Makes OpenID provider login box modal.

---
 lms/templates/provider_login.html | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/lms/templates/provider_login.html b/lms/templates/provider_login.html
index 73d35c019b..620e0c4191 100644
--- a/lms/templates/provider_login.html
+++ b/lms/templates/provider_login.html
@@ -1,14 +1,15 @@
 <%inherit file="main.html" />
 <%namespace name='static' file='static_content.html'/>
 
-<style>
-
+<%block name="headextra">
+<style type="text/css">
 .openid-login {
     display: block;
     position: relative;
     left: 0;
     margin: 100px auto;
     top: 0;
+    z-index: 200;
 }
 
 .openid-login input[type=submit] {
@@ -16,7 +17,17 @@
     height: auto !important;
 }
 
+#lean_overlay {
+  display: block;
+  position: fixed;
+  left: 0px;
+  top: 0px;
+  z-index: 100;
+  width:100%;
+  height:100%;
+}
 </style>
+</%block>
 
 <section id="login-modal" class="modal login-modal openid-login">
   <div class="inner-wrapper">
@@ -38,3 +49,4 @@
     </form>
   </div>
 </section>
+<div id="lean_overlay"></div>

From a9f624115c0f5a08ad95c0c9fe547c777e824bed Mon Sep 17 00:00:00 2001
From: Kyle Fiedler <kfiedler@thoughtbot.com>
Date: Thu, 23 Aug 2012 10:10:13 -0400
Subject: [PATCH 101/181] Added styles for hints

---
 .../lib/xmodule/xmodule/css/capa/display.scss | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/common/lib/xmodule/xmodule/css/capa/display.scss b/common/lib/xmodule/xmodule/css/capa/display.scss
index e6ebdb316f..e7b2250f02 100644
--- a/common/lib/xmodule/xmodule/css/capa/display.scss
+++ b/common/lib/xmodule/xmodule/css/capa/display.scss
@@ -280,4 +280,42 @@ section.problem {
       @extend .blue-button;
     }
   }
+
+  .hints {
+    border: 1px solid #ccc;
+
+    h3 {
+      border-bottom: 1px solid #e3e3e3;
+      text-shadow: 0 1px 0 #fff;
+      padding: 9px;
+      background: #eee;
+      font-weight: bold;
+      font-size: em(16);
+    }
+
+    div {
+      border-bottom: 1px solid #ddd;
+
+      &:last-child {
+        border-bottom: none;
+      }
+
+      p {
+        margin-bottom: 0;
+      }
+
+      header {
+        a {
+          display: block;
+          padding: 9px;
+          background: #f3f3f3;
+          @include box-shadow(inset 0 0 0 1px #fff);
+        }
+      }
+
+      section {
+        padding: 9px;
+      }
+    }
+  }
 }

From 3be48e5012e0bdffd94f510128f5d5f035ae9555 Mon Sep 17 00:00:00 2001
From: Kyle Fiedler <kfiedler@thoughtbot.com>
Date: Thu, 23 Aug 2012 16:12:09 -0400
Subject: [PATCH 102/181] Formatted new hints and testing blocks

---
 common/lib/capa/capa/templates/textbox.html   |  67 +++++----
 .../lib/xmodule/xmodule/css/capa/display.scss | 130 +++++++++++++++++-
 2 files changed, 169 insertions(+), 28 deletions(-)

diff --git a/common/lib/capa/capa/templates/textbox.html b/common/lib/capa/capa/templates/textbox.html
index 19c43482a8..7800e6031e 100644
--- a/common/lib/capa/capa/templates/textbox.html
+++ b/common/lib/capa/capa/templates/textbox.html
@@ -7,26 +7,28 @@
 
   <span id="answer_${id}"></span>
 
-  % if state == 'unsubmitted':
-  <span class="unanswered" style="display:inline-block;" id="status_${id}"></span>
-  % elif state == 'correct':
-  <span class="correct" id="status_${id}"></span>
-  % elif state == 'incorrect':
-  <span class="incorrect" id="status_${id}"></span>
-  % elif state == 'queued':
-  <span class="processing" id="status_${id}"></span>
-  <span style="display:none;" class="xqueue" id="${id}" >${queue_len}</span>
-  % endif
-  % if hidden:
-  <div style="display:none;" name="${hidden}" inputid="input_${id}" />
-  % endif
-  <br/>
-  <span style="display:none;" class="debug">(${state})</span>
-  <br/>
-  <span class="message">${msg|n}</span>
-  <br/>
+  <div class="grader-status">
+    % if state == 'unsubmitted':
+      <span class="unanswered" style="display:inline-block;" id="status_${id}">Unanswered</span>
+    % elif state == 'correct':
+      <span class="correct" id="status_${id}">Correct</span>
+    % elif state == 'incorrect':
+      <span class="incorrect" id="status_${id}">Incorrect</span>
+    % elif state == 'queued':
+      <span class="processing" id="status_${id}">Queued</span>
+      <span style="display:none;" class="xqueue" id="${id}" >${queue_len}</span>
+    % endif
 
-  <br/>
+    % if hidden:
+      <div style="display:none;" name="${hidden}" inputid="input_${id}" />
+    % endif
+
+    <p class="debug">${state}</p>
+  </div>
+
+  <div class="external-grader-message">
+    ${msg|n}
+  </div>
 
   <script>
     // Note: We need to make the area follow the CodeMirror for this to work.
@@ -43,14 +45,25 @@
         indentWithTabs: true,
         smartIndent: false
       });
+
+      // this should be brought back into problems
+      $('.longform-body').hide();
+      $('.longform-header').append('<a href="" class="full">See full output</a>');
+
+      $('.full').click(function() {
+        $(this).parent().siblings().slideToggle();
+        $(this).parent().parent().toggleClass('open');
+        var text = $(this).text() == 'See full output' ? 'Hide output' : 'See full output';
+        $(this).text(text);
+        return false;
+      });
+
+      $('.collapsible section').hide()
+      $('.collapsible header a').click(function() {
+        $(this).parent().siblings().slideToggle();
+        $(this).parent().parent().toggleClass('open');
+        return false
+      });
     });
   </script>
-  <style type="text/css">
-    .CodeMirror {
-        border: 1px solid black;
-        font-size: 14px;
-        line-height: 18px;
-        resize: both;
-    }
-  </style>
 </section>
diff --git a/common/lib/xmodule/xmodule/css/capa/display.scss b/common/lib/xmodule/xmodule/css/capa/display.scss
index e7b2250f02..cca951ce08 100644
--- a/common/lib/xmodule/xmodule/css/capa/display.scss
+++ b/common/lib/xmodule/xmodule/css/capa/display.scss
@@ -16,6 +16,7 @@ h2 {
   }
 }
 
+
 section.problem {
   @media print {
     display: block;
@@ -171,6 +172,30 @@ section.problem {
         top: 6px;
       }
     }
+
+    .grader-status {
+      padding: 9px;
+      background: #F6F6F6;
+      border: 1px solid #ddd;
+      border-top: 0;
+      margin-bottom: 20px;
+      @include clearfix;
+
+      span {
+        text-indent: -9999px;
+        overflow: hidden;
+        display: block;
+        float: left;
+        margin: -7px 7px 0 0;
+      }
+
+      p {
+        line-height: 20px;
+        text-transform: capitalize;
+        margin-bottom: 0;
+        float: left;
+      }
+    }
   }
 
   ul {
@@ -246,6 +271,69 @@ section.problem {
 
   }
 
+  code {
+    margin: 0 2px;
+    padding: 0px 5px;
+    white-space: nowrap;
+    border: 1px solid #EAEAEA;
+    background-color: #F8F8F8;
+    @include border-radius(3px);
+    font-size: .9em;
+  }
+
+  pre {
+    background-color: #F8F8F8;
+    border: 1px solid #CCC;
+    font-size: .9em;
+    line-height: 1.4;
+    overflow: auto;
+    padding: 6px 10px;
+    @include border-radius(3px);
+
+    > code {
+      margin: 0;
+      padding: 0;
+      white-space: pre;
+      border: none;
+      background: transparent;
+    }
+  }
+
+  .CodeMirror {
+    border: 1px solid black;
+    font-size: 14px;
+    line-height: 18px;
+    resize: both;
+
+    pre {
+      @include border-radius(0);
+      border-radius: 0;
+      border-width: 0;
+      margin: 0;
+      padding: 0;
+      background: transparent;
+      font-family: inherit;
+      font-size: inherit;
+      white-space: pre;
+      word-wrap: normal;
+      overflow: hidden;
+      resize: none;
+
+      &.CodeMirror-cursor {
+        z-index: 10;
+        position: absolute;
+        visibility: hidden;
+        border-left: 1px solid black;
+        border-right: none;
+        width: 0;
+      }
+    }
+  }
+
+  .CodeMirror-focused pre.CodeMirror-cursor {
+    visibility: visible;
+  }
+
   hr {
     background: #ddd;
     border: none;
@@ -308,7 +396,7 @@ section.problem {
         a {
           display: block;
           padding: 9px;
-          background: #f3f3f3;
+          background: #F6F6F6;
           @include box-shadow(inset 0 0 0 1px #fff);
         }
       }
@@ -318,4 +406,44 @@ section.problem {
       }
     }
   }
+
+  .test {
+    border-top: 1px solid #aaa;
+    padding-top: 18px;
+
+    header {
+      margin-bottom: 12px;
+    }
+
+    .shortform, .longform {
+      border: 1px solid #d3d3d3;
+      padding: 9px;
+      margin-bottom: 12px;
+      @include box-shadow(inset 0 0 0 1px #f3f3f3);
+
+      p:last-of-type {
+        margin-bottom: 0;
+      }
+    }
+
+    a.full {
+      @include position(absolute,  0 0 0px 0px);
+      font-size: .8em;
+      padding: 4px;
+      text-align: right;
+      width: 100%;
+      display: block;
+      background: #F3F3F3;
+      @include box-sizing(border-box);
+    }
+
+    .longform {
+      position: relative;
+      padding-bottom: 20px;
+    }
+
+    .longform-header {
+      margin-bottom: lh();
+    }
+  }
 }

From bc288424a4cfd808be7575519d1f9e5a13e6ed39 Mon Sep 17 00:00:00 2001
From: Kyle Fiedler <kfiedler@thoughtbot.com>
Date: Thu, 30 Aug 2012 15:22:27 -0400
Subject: [PATCH 103/181] Rework styles for test results and style file upload

---
 .../capa/capa/templates/filesubmission.html   | 19 ++++----
 .../lib/xmodule/xmodule/css/capa/display.scss | 45 ++++++++++++++++---
 .../xmodule/js/src/capa/display.coffee        |  2 +-
 3 files changed, 49 insertions(+), 17 deletions(-)

diff --git a/common/lib/capa/capa/templates/filesubmission.html b/common/lib/capa/capa/templates/filesubmission.html
index e9fd7c5674..630a3222dc 100644
--- a/common/lib/capa/capa/templates/filesubmission.html
+++ b/common/lib/capa/capa/templates/filesubmission.html
@@ -1,17 +1,18 @@
 <section id="filesubmission_${id}" class="filesubmission">
-    <input type="file" name="input_${id}" id="input_${id}" value="${value}" multiple="multiple" data-required_files="${required_files}" data-allowed_files="${allowed_files}"/><br />
+  <div class="grader-status file">
     % if state == 'unsubmitted':
-    <span class="unanswered" style="display:inline-block;" id="status_${id}"></span>
+    <span class="unanswered" style="display:inline-block;" id="status_${id}">Unanswered</span>
     % elif state == 'correct':
-    <span class="correct" id="status_${id}"></span>
+    <span class="correct" id="status_${id}">Correct</span>
     % elif state == 'incorrect':
-    <span class="incorrect" id="status_${id}"></span>
+    <span class="incorrect" id="status_${id}">Incorrect</span>
     % elif state == 'queued':
-    <span class="processing" id="status_${id}"></span>
+    <span class="processing" id="status_${id}">Queued</span>
     <span style="display:none;" class="xqueue" id="${id}" >${queue_len}</span>
     % endif
-    <span style="display:none;" class="debug">(${state})</span>
-    <br/>
-    <span class="message">${msg|n}</span>
-    <br/>
+    <p class="debug">${state}</p>
+
+    <input type="file" name="input_${id}" id="input_${id}" value="${value}" multiple="multiple" data-required_files="${required_files}" data-allowed_files="${allowed_files}"/>
+  </div>
+    <div class="message">${msg|n}</div>
 </section>
diff --git a/common/lib/xmodule/xmodule/css/capa/display.scss b/common/lib/xmodule/xmodule/css/capa/display.scss
index cca951ce08..16c8de41ac 100644
--- a/common/lib/xmodule/xmodule/css/capa/display.scss
+++ b/common/lib/xmodule/xmodule/css/capa/display.scss
@@ -195,9 +195,31 @@ section.problem {
         margin-bottom: 0;
         float: left;
       }
+
+      &.file {
+        background: #FFF;
+        margin-top: 20px;
+        padding: 20px 0 0 0;
+
+        border: {
+          top: 1px solid #eee;
+          right: 0;
+          bottom: 0;
+          left: 0;
+        }
+
+        p.debug {
+          display: none;
+        }
+
+        input {
+          float: left;
+        }
+      }
     }
   }
 
+
   ul {
     list-style: disc outside none;
     margin-bottom: lh();
@@ -408,18 +430,27 @@ section.problem {
   }
 
   .test {
-    border-top: 1px solid #aaa;
     padding-top: 18px;
 
     header {
       margin-bottom: 12px;
+
+      h3 {
+        font-size: 0.9em;
+        font-weight: bold;
+        font-style: normal;
+        text-transform: uppercase;
+        color: #AAA;
+      }
     }
 
-    .shortform, .longform {
-      border: 1px solid #d3d3d3;
+    .longform, .shortform {
+      border: 1px solid #ddd;
       padding: 9px;
-      margin-bottom: 12px;
-      @include box-shadow(inset 0 0 0 1px #f3f3f3);
+      margin-bottom: 10px;
+      background: #FFF;
+      @include box-shadow(inset 0 0 0 1px #eee);
+      @include border-radius(3px);
 
       p:last-of-type {
         margin-bottom: 0;
@@ -427,7 +458,7 @@ section.problem {
     }
 
     a.full {
-      @include position(absolute,  0 0 0px 0px);
+      @include position(absolute,  0 0 1px 0px);
       font-size: .8em;
       padding: 4px;
       text-align: right;
@@ -443,7 +474,7 @@ section.problem {
     }
 
     .longform-header {
-      margin-bottom: lh();
+      margin-bottom: .6em;
     }
   }
 }
diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee
index 0ea6cffb58..24363735ae 100644
--- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee
+++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee
@@ -264,7 +264,7 @@ class @Problem
     alert_elem = "<div class='capa_alert'>" + msg + "</div>"
     @el.find('.action').after(alert_elem)
     @el.find('.capa_alert').animate(opacity: 0, 500).animate(opacity: 1, 500)
-    
+
   save: =>
     Logger.log 'problem_save', @answers
     $.postWithPrefix "#{@url}/problem_save", @answers, (response) =>

From 1939b001e4c194b39a8717a4e0b85d4874ac1aa0 Mon Sep 17 00:00:00 2001
From: Kyle Fiedler <kfiedler@thoughtbot.com>
Date: Thu, 30 Aug 2012 16:24:02 -0400
Subject: [PATCH 104/181] Added better animation in

---
 common/lib/xmodule/xmodule/css/capa/display.scss      | 9 +++++++++
 common/lib/xmodule/xmodule/js/src/capa/display.coffee | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/common/lib/xmodule/xmodule/css/capa/display.scss b/common/lib/xmodule/xmodule/css/capa/display.scss
index 16c8de41ac..e171d35513 100644
--- a/common/lib/xmodule/xmodule/css/capa/display.scss
+++ b/common/lib/xmodule/xmodule/css/capa/display.scss
@@ -391,6 +391,15 @@ section.problem {
     }
   }
 
+  div.capa_alert {
+    padding: 8px 12px;
+    border: 1px solid #EBE8BF;
+    border-radius: 3px;
+    background: #FFFCDD;
+    font-size: 0.9em;
+    margin-top: 10px;
+  }
+
   .hints {
     border: 1px solid #ccc;
 
diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee
index 24363735ae..098b79d9cf 100644
--- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee
+++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee
@@ -263,7 +263,7 @@ class @Problem
       @el.find('.capa_alert').remove()
     alert_elem = "<div class='capa_alert'>" + msg + "</div>"
     @el.find('.action').after(alert_elem)
-    @el.find('.capa_alert').animate(opacity: 0, 500).animate(opacity: 1, 500)
+    @el.find('.capa_alert').css(opacity: 0).animate(opacity: 1, 700)
 
   save: =>
     Logger.log 'problem_save', @answers

From 22aa325d83a3fcf7680f00fc4e4ae67873ca4099 Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Thu, 30 Aug 2012 16:37:40 -0400
Subject: [PATCH 105/181] Enable linking into the middle of sequences

* add a url that has an extra /{position} at the end
* pass it through to get_module
---
 lms/djangoapps/courseware/module_render.py | 3 +--
 lms/djangoapps/courseware/views.py         | 4 +---
 lms/urls.py                                | 4 +++-
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py
index 82ec3cadeb..ee29491d27 100644
--- a/lms/djangoapps/courseware/module_render.py
+++ b/lms/djangoapps/courseware/module_render.py
@@ -195,7 +195,6 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
                                                         descriptor.category,
                                                         shared_state_key)
 
-
     instance_state = instance_module.state if instance_module is not None else None
     shared_state = shared_module.state if shared_module is not None else None
 
@@ -254,7 +253,7 @@ def _get_module(user, request, location, student_module_cache, course_id, positi
                           # by the replace_static_urls code below
                           replace_urls=replace_urls,
                           node_path=settings.NODE_PATH,
-                          anonymous_student_id=anonymous_student_id
+                          anonymous_student_id=anonymous_student_id,
                           )
     # pass position specified in URL to module through ModuleSystem
     system.set('position', position)
diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py
index 92f6716320..5e75e99a4f 100644
--- a/lms/djangoapps/courseware/views.py
+++ b/lms/djangoapps/courseware/views.py
@@ -5,8 +5,6 @@ import itertools
 
 from functools import partial
 
-from functools import partial
-
 from django.conf import settings
 from django.core.context_processors import csrf
 from django.core.urlresolvers import reverse
@@ -152,7 +150,7 @@ def index(request, course_id, chapter=None, section=None,
                     course_id, request.user, section_descriptor)
                 module = get_module(request.user, request,
                                     section_descriptor.location,
-                                    student_module_cache, course_id)
+                                    student_module_cache, course_id, position)
                 if module is None:
                     # User is probably being clever and trying to access something
                     # they don't have access to.
diff --git a/lms/urls.py b/lms/urls.py
index 86d654eb40..f19f4682f9 100644
--- a/lms/urls.py
+++ b/lms/urls.py
@@ -142,6 +142,8 @@ if settings.COURSEWARE_ENABLED:
             'courseware.views.index', name="courseware_chapter"),
         url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/courseware/(?P<chapter>[^/]*)/(?P<section>[^/]*)/$',
             'courseware.views.index', name="courseware_section"),
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/courseware/(?P<chapter>[^/]*)/(?P<section>[^/]*)/(?P<position>[^/]*)/?$',
+            'courseware.views.index', name="courseware_position"),
         url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/progress$',
             'courseware.views.progress', name="progress"),
         # Takes optional student_id for instructor use--shows profile as that student sees it.
@@ -164,7 +166,7 @@ if settings.COURSEWARE_ENABLED:
     if settings.MITX_FEATURES.get('ENABLE_DISCUSSION_SERVICE'):
 
         urlpatterns += (
-            url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/news$', 
+            url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/news$',
                 'courseware.views.news', name="news"),
             url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/discussion/',
                 include('django_comment_client.urls'))

From 021ccde1b41610363664ea110d228e46210608d1 Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Thu, 30 Aug 2012 18:47:11 -0400
Subject: [PATCH 106/181] Make jump_to work with the xml modulestore

* it now works in the context of a specific course_id
* add tracking of parent locations to xml modulestore
* adjust lots of tests, including some refactoring
* NOT working yet: jumping to the right position in a sequence.
---
 .../lib/xmodule/xmodule/modulestore/search.py | 12 ++--
 .../xmodule/modulestore/tests/__init__.py     | 12 ++++
 .../modulestore/tests/test_modulestore.py     | 34 ++++++++++
 .../xmodule/modulestore/tests/test_mongo.py   | 37 +---------
 .../xmodule/modulestore/tests/test_xml.py     | 16 +++++
 common/lib/xmodule/xmodule/modulestore/xml.py | 67 +++++++++++++++++--
 common/lib/xmodule/xmodule/x_module.py        |  8 ++-
 lms/djangoapps/courseware/tests/tests.py      |  6 +-
 lms/djangoapps/courseware/views.py            |  4 +-
 lms/urls.py                                   |  3 +-
 10 files changed, 149 insertions(+), 50 deletions(-)
 create mode 100644 common/lib/xmodule/xmodule/modulestore/tests/__init__.py
 create mode 100644 common/lib/xmodule/xmodule/modulestore/tests/test_modulestore.py
 create mode 100644 common/lib/xmodule/xmodule/modulestore/tests/test_xml.py

diff --git a/common/lib/xmodule/xmodule/modulestore/search.py b/common/lib/xmodule/xmodule/modulestore/search.py
index baf3d46b57..a1ed407ee2 100644
--- a/common/lib/xmodule/xmodule/modulestore/search.py
+++ b/common/lib/xmodule/xmodule/modulestore/search.py
@@ -6,15 +6,14 @@ from .exceptions import (ItemNotFoundError, NoPathToItem)
 from . import ModuleStore, Location
 
 
-def path_to_location(modulestore, location, course_name=None):
+def path_to_location(modulestore, course_id, location):
     '''
     Try to find a course_id/chapter/section[/position] path to location in
     modulestore.  The courseware insists that the first level in the course is
     chapter, but any kind of module can be a "section".
 
     location: something that can be passed to Location
-    course_name: [optional].  If not None, restrict search to paths
-        in that course.
+    course_id: Search for paths in this course.
 
     raise ItemNotFoundError if the location doesn't exist.
 
@@ -41,7 +40,7 @@ def path_to_location(modulestore, location, course_name=None):
             xs = xs[1]
         return p
 
-    def find_path_to_course(location, course_name=None):
+    def find_path_to_course():
         '''Find a path up the location graph to a node with the
         specified category.
 
@@ -69,7 +68,8 @@ def path_to_location(modulestore, location, course_name=None):
 
             # print 'Processing loc={0}, path={1}'.format(loc, path)
             if loc.category == "course":
-                if course_name is None or course_name == loc.name:
+                # confirm that this is the right course
+                if course_id == CourseDescriptor.location_to_id(loc):
                     # Found it!
                     path = (loc, path)
                     return flatten(path)
@@ -81,7 +81,7 @@ def path_to_location(modulestore, location, course_name=None):
         # If we're here, there is no path
         return None
 
-    path = find_path_to_course(location, course_name)
+    path = find_path_to_course()
     if path is None:
         raise(NoPathToItem(location))
 
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/__init__.py b/common/lib/xmodule/xmodule/modulestore/tests/__init__.py
new file mode 100644
index 0000000000..126f0136e2
--- /dev/null
+++ b/common/lib/xmodule/xmodule/modulestore/tests/__init__.py
@@ -0,0 +1,12 @@
+from path import path
+
+# from ~/mitx_all/mitx/common/lib/xmodule/xmodule/modulestore/tests/
+# to   ~/mitx_all/mitx/common/test
+TEST_DIR = path(__file__).abspath().dirname()
+for i in range(5):
+    TEST_DIR = TEST_DIR.dirname()
+TEST_DIR = TEST_DIR / 'test'
+
+DATA_DIR = TEST_DIR / 'data'
+
+
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_modulestore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_modulestore.py
new file mode 100644
index 0000000000..c1d1d50a53
--- /dev/null
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_modulestore.py
@@ -0,0 +1,34 @@
+from nose.tools import assert_equals, assert_raises, assert_not_equals, with_setup
+
+from xmodule.modulestore.exceptions import ItemNotFoundError, NoPathToItem
+from xmodule.modulestore.search import path_to_location
+
+def check_path_to_location(modulestore):
+    '''Make sure that path_to_location works: should be passed a modulestore
+    with the toy and simple courses loaded.'''
+    should_work = (
+        ("i4x://edX/toy/video/Welcome",
+         ("edX/toy/2012_Fall", "Overview", "Welcome", None)),
+        ("i4x://edX/toy/chapter/Overview",
+         ("edX/toy/2012_Fall", "Overview", None, None)),
+        )
+    course_id = "edX/toy/2012_Fall"
+
+    for location, expected in should_work:
+        assert_equals(path_to_location(modulestore, course_id, location), expected)
+
+    not_found = (
+        "i4x://edX/toy/video/WelcomeX", "i4x://edX/toy/course/NotHome"
+        )
+    for location in not_found:
+        assert_raises(ItemNotFoundError, path_to_location, modulestore, course_id, location)
+
+    # Since our test files are valid, there shouldn't be any
+    # elements with no path to them.  But we can look for them in
+    # another course.
+    no_path = (
+        "i4x://edX/simple/video/Lost_Video",
+        )
+    for location in no_path:
+        assert_raises(NoPathToItem, path_to_location, modulestore, course_id, location)
+
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
index 746240e763..4c593e391e 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
@@ -1,23 +1,14 @@
 import pymongo
 
 from nose.tools import assert_equals, assert_raises, assert_not_equals, with_setup
-from path import path
 from pprint import pprint
 
 from xmodule.modulestore import Location
-from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, NoPathToItem
 from xmodule.modulestore.mongo import MongoModuleStore
 from xmodule.modulestore.xml_importer import import_from_xml
-from xmodule.modulestore.search import path_to_location
 
-# from ~/mitx_all/mitx/common/lib/xmodule/xmodule/modulestore/tests/
-# to   ~/mitx_all/mitx/common/test
-TEST_DIR = path(__file__).abspath().dirname()
-for i in range(5):
-    TEST_DIR = TEST_DIR.dirname()
-TEST_DIR = TEST_DIR / 'test'
-
-DATA_DIR = TEST_DIR / 'data'
+from .test_modulestore import check_path_to_location
+from . import DATA_DIR
 
 
 HOST = 'localhost'
@@ -110,27 +101,5 @@ class TestMongoModuleStore(object):
 
     def test_path_to_location(self):
         '''Make sure that path_to_location works'''
-        should_work = (
-            ("i4x://edX/toy/video/Welcome",
-             ("edX/toy/2012_Fall", "Overview", "Welcome", None)),
-            ("i4x://edX/toy/chapter/Overview",
-             ("edX/toy/2012_Fall", "Overview", None, None)),
-            )
-        for location, expected in should_work:
-            assert_equals(path_to_location(self.store, location), expected)
-
-        not_found = (
-            "i4x://edX/toy/video/WelcomeX", "i4x://edX/toy/course/NotHome"
-            )
-        for location in not_found:
-            assert_raises(ItemNotFoundError, path_to_location, self.store, location)
-
-        # Since our test files are valid, there shouldn't be any
-        # elements with no path to them.  But we can look for them in
-        # another course.
-        no_path = (
-            "i4x://edX/simple/video/Lost_Video",
-            )
-        for location in no_path:
-            assert_raises(NoPathToItem, path_to_location, self.store, location, "toy")
+        check_path_to_location(self.store)
 
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_xml.py b/common/lib/xmodule/xmodule/modulestore/tests/test_xml.py
new file mode 100644
index 0000000000..0c5c212bbf
--- /dev/null
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_xml.py
@@ -0,0 +1,16 @@
+from xmodule.modulestore import Location
+from xmodule.modulestore.xml import XMLModuleStore
+from xmodule.modulestore.xml_importer import import_from_xml
+
+from .test_modulestore import check_path_to_location
+from . import DATA_DIR
+
+class TestXMLModuleStore(object):
+    def test_path_to_location(self):
+        """Make sure that path_to_location works properly"""
+
+        print "Starting import"
+        modulestore = XMLModuleStore(DATA_DIR, eager=True, course_dirs=['toy', 'simple'])
+        print "finished import"
+        
+        check_path_to_location(modulestore)
diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py
index 92eca8f5e6..ce3635c4b9 100644
--- a/common/lib/xmodule/xmodule/modulestore/xml.py
+++ b/common/lib/xmodule/xmodule/modulestore/xml.py
@@ -37,7 +37,7 @@ def clean_out_mako_templating(xml_string):
 
 class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
     def __init__(self, xmlstore, course_id, course_dir,
-                 policy, error_tracker, **kwargs):
+                 policy, error_tracker, parent_tracker, **kwargs):
         """
         A class that handles loading from xml.  Does some munging to ensure that
         all elements have unique slugs.
@@ -134,8 +134,8 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
 
             xmlstore.modules[course_id][descriptor.location] = descriptor
 
-            if xmlstore.eager:
-                descriptor.get_children()
+            for child in descriptor.get_children():
+                parent_tracker.add_parent(child.location, descriptor.location)
             return descriptor
 
         render_template = lambda: ''
@@ -151,6 +151,46 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                                   error_tracker, process_xml, policy, **kwargs)
 
 
+class ParentTracker(object):
+    """A simple class to factor out the logic for tracking location parent pointers."""
+    def __init__(self):
+        """
+        Init
+        """
+        # location -> set(parents).  Not using defaultdict because we care about the empty case.
+        self._parents = dict()
+
+    def add_parent(self, child, parent):
+        """
+        Add a parent of child location to the set of parents.  Duplicate calls have no effect.
+
+        child and parent must be something that can be passed to Location.
+        """
+        child = Location(child)
+        parent = Location(parent)
+        s = self._parents.setdefault(child, set())
+        s.add(parent)
+
+    def is_known(self, child):
+        """
+        returns True iff child has some parents.
+        """
+        child = Location(child)
+        return child in self._parents
+
+    def make_known(self, location):
+        """Tell the parent tracker about an object, without registering any
+        parents for it.  Used for the top level course descriptor locations."""
+        self._parents.setdefault(location, set())
+
+    def parents(self, child):
+        """
+        Return a list of the parents of this child.  If not is_known(child), will throw a KeyError
+        """
+        child = Location(child)
+        return list(self._parents[child])
+
+
 class XMLModuleStore(ModuleStoreBase):
     """
     An XML backed ModuleStore
@@ -191,6 +231,8 @@ class XMLModuleStore(ModuleStoreBase):
         #log.debug('XMLModuleStore: eager=%s, data_dir = %s' % (eager, self.data_dir))
         #log.debug('default_class = %s' % self.default_class)
 
+        self.parent_tracker = ParentTracker()
+
         # If we are specifically asked for missing courses, that should
         # be an error.  If we are asked for "all" courses, find the ones
         # that have a course.xml
@@ -221,6 +263,7 @@ class XMLModuleStore(ModuleStoreBase):
         if course_descriptor is not None:
             self.courses[course_dir] = course_descriptor
             self._location_errors[course_descriptor.location] = errorlog
+            self.parent_tracker.make_known(course_descriptor.location)
         else:
             # Didn't load course.  Instead, save the errors elsewhere.
             self.errored_courses[course_dir] = errorlog
@@ -339,7 +382,7 @@ class XMLModuleStore(ModuleStoreBase):
 
 
             course_id = CourseDescriptor.make_id(org, course, url_name)
-            system = ImportSystem(self, course_id, course_dir, policy, tracker)
+            system = ImportSystem(self, course_id, course_dir, policy, tracker, self.parent_tracker)
 
             course_descriptor = system.process_xml(etree.tostring(course_data))
 
@@ -450,3 +493,19 @@ class XMLModuleStore(ModuleStoreBase):
         metadata: A nested dictionary of module metadata
         """
         raise NotImplementedError("XMLModuleStores are read-only")
+
+    def get_parent_locations(self, location):
+        '''Find all locations that are the parents of this location.  Needed
+        for path_to_location().
+
+        If there is no data at location in this modulestore, raise
+            ItemNotFoundError.
+
+        returns an iterable of things that can be passed to Location.  This may
+        be empty if there are no parents.
+        '''
+        location = Location.ensure_fully_specified(location)
+        if not self.parent_tracker.is_known(location):
+            raise ItemNotFoundError(location)
+
+        return self.parent_tracker.parents(location)
diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py
index 0dc16bd976..82f623e977 100644
--- a/common/lib/xmodule/xmodule/x_module.py
+++ b/common/lib/xmodule/xmodule/x_module.py
@@ -544,7 +544,13 @@ class XModuleDescriptor(Plugin, HTMLSnippet):
             # Put import here to avoid circular import errors
             from xmodule.error_module import ErrorDescriptor
             msg = "Error loading from xml."
-            log.warning(msg + " " + str(err))
+            log.warning(msg + " " + str(err)[:200])
+
+            # Normally, we don't want lots of exception traces in our logs from common
+            # content problems.  But if you're debugging the xml loading code itself,
+            # uncomment the next line.
+            # log.exception(msg)
+
             system.error_tracker(msg)
             err_msg = msg + "\n" + exc_info_to_str(sys.exc_info())
             descriptor = ErrorDescriptor.from_xml(xml_data, system, org, course,
diff --git a/lms/djangoapps/courseware/tests/tests.py b/lms/djangoapps/courseware/tests/tests.py
index 88e0f53f70..280dada391 100644
--- a/lms/djangoapps/courseware/tests/tests.py
+++ b/lms/djangoapps/courseware/tests/tests.py
@@ -204,7 +204,8 @@ class PageLoader(ActivateLoginTestCase):
         self.assertEqual(len(courses), 1)
         course = courses[0]
         self.enroll(course)
-
+        course_id = course.id
+        
         n = 0
         num_bad = 0
         all_ok = True
@@ -214,7 +215,8 @@ class PageLoader(ActivateLoginTestCase):
             print "Checking ", descriptor.location.url()
             #print descriptor.__class__, descriptor.location
             resp = self.client.get(reverse('jump_to',
-                                   kwargs={'location': descriptor.location.url()}))
+                                   kwargs={'course_id': course_id,
+                                           'location': descriptor.location.url()}))
             msg = str(resp.status_code)
 
             if resp.status_code != 200:
diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py
index 5e75e99a4f..3f0f54147b 100644
--- a/lms/djangoapps/courseware/views.py
+++ b/lms/djangoapps/courseware/views.py
@@ -194,7 +194,7 @@ def index(request, course_id, chapter=None, section=None,
 
 
 @ensure_csrf_cookie
-def jump_to(request, location):
+def jump_to(request, course_id, location):
     '''
     Show the page that contains a specific location.
 
@@ -211,7 +211,7 @@ def jump_to(request, location):
 
     # Complain if there's not data for this location
     try:
-        (course_id, chapter, section, position) = path_to_location(modulestore(), location)
+        (course_id, chapter, section, position) = path_to_location(modulestore(), course_id, location)
     except ItemNotFoundError:
         raise Http404("No data at this location: {0}".format(location))
     except NoPathToItem:
diff --git a/lms/urls.py b/lms/urls.py
index f19f4682f9..2fecaa94cc 100644
--- a/lms/urls.py
+++ b/lms/urls.py
@@ -98,8 +98,9 @@ if settings.COURSEWARE_ENABLED:
     urlpatterns += (
         # Hook django-masquerade, allowing staff to view site as other users
         url(r'^masquerade/', include('masquerade.urls')),
-        url(r'^jump_to/(?P<location>.*)$', 'courseware.views.jump_to', name="jump_to"),
 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/jump_to/(?P<location>.*)$',
+            'courseware.views.jump_to', name="jump_to"),
         url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/modx/(?P<location>.*?)/(?P<dispatch>[^/]*)$',
             'courseware.module_render.modx_dispatch',
             name='modx_dispatch'),

From 484735bf034d5190a3e8064eb51e4b4ea252f4fd Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Thu, 30 Aug 2012 18:55:38 -0400
Subject: [PATCH 107/181] make jump_to go to right position in a sequence.

---
 common/lib/xmodule/xmodule/modulestore/search.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/search.py b/common/lib/xmodule/xmodule/modulestore/search.py
index a1ed407ee2..87f1cbf825 100644
--- a/common/lib/xmodule/xmodule/modulestore/search.py
+++ b/common/lib/xmodule/xmodule/modulestore/search.py
@@ -26,7 +26,7 @@ def path_to_location(modulestore, course_id, location):
     A location may be accessible via many paths. This method may
     return any valid path.
 
-    If the section is a sequence, position will be the position
+    If the section is a sequential or vertical, position will be the position
     of this location in that sequence.  Otherwise, position will
     be None. TODO (vshnayder): Not true yet.
     '''
@@ -83,15 +83,21 @@ def path_to_location(modulestore, course_id, location):
 
     path = find_path_to_course()
     if path is None:
-        raise(NoPathToItem(location))
+        raise NoPathToItem(location)
 
     n = len(path)
     course_id = CourseDescriptor.location_to_id(path[0])
     # pull out the location names
     chapter = path[1].name if n > 1 else None
     section = path[2].name if n > 2 else None
-
-    # TODO (vshnayder): not handling position at all yet...
+    # Figure out the position
     position = None
+    if n > 3:
+        section_desc = modulestore.get_instance(course_id, path[2])
+        child_locs = [c.location for c in section_desc.get_children()]
+        # positions are 1-indexed, and should be strings to be consistent with
+        # url parsing.
+        position = str(child_locs.index(path[3]) + 1)
+
 
     return (course_id, chapter, section, position)

From d815d50aee5d8660944a8ec37ae42f64612f7beb Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Thu, 30 Aug 2012 19:26:50 -0400
Subject: [PATCH 108/181] Remove obsolete eager flag for xml modulestore

* has to be eager
---
 .../xmodule/xmodule/modulestore/tests/test_xml.py    |  2 +-
 common/lib/xmodule/xmodule/modulestore/xml.py        | 12 +-----------
 .../lib/xmodule/xmodule/modulestore/xml_importer.py  |  3 +--
 common/lib/xmodule/xmodule/tests/test_export.py      |  4 ++--
 common/lib/xmodule/xmodule/tests/test_import.py      |  8 ++++----
 .../courseware/management/commands/clean_xml.py      |  1 -
 .../management/commands/metadata_to_json.py          |  1 -
 lms/djangoapps/courseware/tests/tests.py             |  1 -
 lms/djangoapps/django_comment_client/utils.py        |  9 +++++----
 lms/envs/common.py                                   |  1 -
 10 files changed, 14 insertions(+), 28 deletions(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_xml.py b/common/lib/xmodule/xmodule/modulestore/tests/test_xml.py
index 0c5c212bbf..c4446bebb5 100644
--- a/common/lib/xmodule/xmodule/modulestore/tests/test_xml.py
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_xml.py
@@ -10,7 +10,7 @@ class TestXMLModuleStore(object):
         """Make sure that path_to_location works properly"""
 
         print "Starting import"
-        modulestore = XMLModuleStore(DATA_DIR, eager=True, course_dirs=['toy', 'simple'])
+        modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy', 'simple'])
         print "finished import"
         
         check_path_to_location(modulestore)
diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py
index ce3635c4b9..8cb3da7e7a 100644
--- a/common/lib/xmodule/xmodule/modulestore/xml.py
+++ b/common/lib/xmodule/xmodule/modulestore/xml.py
@@ -195,8 +195,7 @@ class XMLModuleStore(ModuleStoreBase):
     """
     An XML backed ModuleStore
     """
-    def __init__(self, data_dir, default_class=None, eager=False,
-                 course_dirs=None):
+    def __init__(self, data_dir, default_class=None, course_dirs=None):
         """
         Initialize an XMLModuleStore from data_dir
 
@@ -205,15 +204,11 @@ class XMLModuleStore(ModuleStoreBase):
         default_class: dot-separated string defining the default descriptor
             class to use if none is specified in entry_points
 
-        eager: If true, load the modules children immediately to force the
-            entire course tree to be parsed
-
         course_dirs: If specified, the list of course_dirs to load. Otherwise,
             load all course dirs
         """
         ModuleStoreBase.__init__(self)
 
-        self.eager = eager
         self.data_dir = path(data_dir)
         self.modules = defaultdict(dict)  # course_id -> dict(location -> XModuleDescriptor)
         self.courses = {}  # course_dir -> XModuleDescriptor for the course
@@ -226,11 +221,6 @@ class XMLModuleStore(ModuleStoreBase):
             class_ = getattr(import_module(module_path), class_name)
             self.default_class = class_
 
-        # TODO (cpennington): We need a better way of selecting specific sets of
-        # debug messages to enable. These were drowning out important messages
-        #log.debug('XMLModuleStore: eager=%s, data_dir = %s' % (eager, self.data_dir))
-        #log.debug('default_class = %s' % self.default_class)
-
         self.parent_tracker = ParentTracker()
 
         # If we are specifically asked for missing courses, that should
diff --git a/common/lib/xmodule/xmodule/modulestore/xml_importer.py b/common/lib/xmodule/xmodule/modulestore/xml_importer.py
index 89f94d8cdb..be0bdc24c2 100644
--- a/common/lib/xmodule/xmodule/modulestore/xml_importer.py
+++ b/common/lib/xmodule/xmodule/modulestore/xml_importer.py
@@ -6,7 +6,7 @@ from .exceptions import DuplicateItemError
 log = logging.getLogger(__name__)
 
 
-def import_from_xml(store, data_dir, course_dirs=None, eager=True,
+def import_from_xml(store, data_dir, course_dirs=None, 
                     default_class='xmodule.raw_module.RawDescriptor'):
     """
     Import the specified xml data_dir into the "store" modulestore,
@@ -19,7 +19,6 @@ def import_from_xml(store, data_dir, course_dirs=None, eager=True,
     module_store = XMLModuleStore(
         data_dir,
         default_class=default_class,
-        eager=eager,
         course_dirs=course_dirs
     )
     for course_id in module_store.modules.keys():
diff --git a/common/lib/xmodule/xmodule/tests/test_export.py b/common/lib/xmodule/xmodule/tests/test_export.py
index 2520d95937..826e6c9d5a 100644
--- a/common/lib/xmodule/xmodule/tests/test_export.py
+++ b/common/lib/xmodule/xmodule/tests/test_export.py
@@ -49,7 +49,7 @@ class RoundTripTestCase(unittest.TestCase):
         copytree(data_dir / course_dir, root_dir / course_dir)
 
         print "Starting import"
-        initial_import = XMLModuleStore(root_dir, eager=True, course_dirs=[course_dir])
+        initial_import = XMLModuleStore(root_dir, course_dirs=[course_dir])
 
         courses = initial_import.get_courses()
         self.assertEquals(len(courses), 1)
@@ -66,7 +66,7 @@ class RoundTripTestCase(unittest.TestCase):
             course_xml.write(xml)
 
         print "Starting second import"
-        second_import = XMLModuleStore(root_dir, eager=True, course_dirs=[course_dir])
+        second_import = XMLModuleStore(root_dir, course_dirs=[course_dir])
 
         courses2 = second_import.get_courses()
         self.assertEquals(len(courses2), 1)
diff --git a/common/lib/xmodule/xmodule/tests/test_import.py b/common/lib/xmodule/xmodule/tests/test_import.py
index a369850209..e81d82bf9e 100644
--- a/common/lib/xmodule/xmodule/tests/test_import.py
+++ b/common/lib/xmodule/xmodule/tests/test_import.py
@@ -193,7 +193,7 @@ class ImportTestCase(unittest.TestCase):
         """Make sure that metadata is inherited properly"""
 
         print "Starting import"
-        initial_import = XMLModuleStore(DATA_DIR, eager=True, course_dirs=['toy'])
+        initial_import = XMLModuleStore(DATA_DIR, course_dirs=['toy'])
 
         courses = initial_import.get_courses()
         self.assertEquals(len(courses), 1)
@@ -216,7 +216,7 @@ class ImportTestCase(unittest.TestCase):
         def get_course(name):
             print "Importing {0}".format(name)
 
-            modulestore = XMLModuleStore(DATA_DIR, eager=True, course_dirs=[name])
+            modulestore = XMLModuleStore(DATA_DIR, course_dirs=[name])
             courses = modulestore.get_courses()
             self.assertEquals(len(courses), 1)
             return courses[0]
@@ -245,7 +245,7 @@ class ImportTestCase(unittest.TestCase):
         happen--locations should uniquely name definitions.  But in
         our imperfect XML world, it can (and likely will) happen."""
 
-        modulestore = XMLModuleStore(DATA_DIR, eager=True, course_dirs=['toy', 'two_toys'])
+        modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy', 'two_toys'])
 
         toy_id = "edX/toy/2012_Fall"
         two_toy_id = "edX/toy/TT_2012_Fall"
@@ -261,7 +261,7 @@ class ImportTestCase(unittest.TestCase):
         """Ensure that colons in url_names convert to file paths properly"""
 
         print "Starting import"
-        modulestore = XMLModuleStore(DATA_DIR, eager=True, course_dirs=['toy'])
+        modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy'])
 
         courses = modulestore.get_courses()
         self.assertEquals(len(courses), 1)
diff --git a/lms/djangoapps/courseware/management/commands/clean_xml.py b/lms/djangoapps/courseware/management/commands/clean_xml.py
index 9fd52178a2..425dd156c1 100644
--- a/lms/djangoapps/courseware/management/commands/clean_xml.py
+++ b/lms/djangoapps/courseware/management/commands/clean_xml.py
@@ -57,7 +57,6 @@ def import_with_checks(course_dir, verbose=True):
     # module.
     modulestore = XMLModuleStore(data_dir,
                    default_class=None,
-                   eager=True,
                    course_dirs=course_dirs)
 
     def str_of_err(tpl):
diff --git a/lms/djangoapps/courseware/management/commands/metadata_to_json.py b/lms/djangoapps/courseware/management/commands/metadata_to_json.py
index dcbcdc0df3..8ef0dee7b3 100644
--- a/lms/djangoapps/courseware/management/commands/metadata_to_json.py
+++ b/lms/djangoapps/courseware/management/commands/metadata_to_json.py
@@ -23,7 +23,6 @@ def import_course(course_dir, verbose=True):
     # module.
     modulestore = XMLModuleStore(data_dir,
                    default_class=None,
-                   eager=True,
                    course_dirs=course_dirs)
 
     def str_of_err(tpl):
diff --git a/lms/djangoapps/courseware/tests/tests.py b/lms/djangoapps/courseware/tests/tests.py
index 280dada391..56a2384bc5 100644
--- a/lms/djangoapps/courseware/tests/tests.py
+++ b/lms/djangoapps/courseware/tests/tests.py
@@ -68,7 +68,6 @@ def xml_store_config(data_dir):
         'OPTIONS': {
             'data_dir': data_dir,
             'default_class': 'xmodule.hidden_module.HiddenDescriptor',
-            'eager': True,
         }
     }
 }
diff --git a/lms/djangoapps/django_comment_client/utils.py b/lms/djangoapps/django_comment_client/utils.py
index 516344d79b..1122db3ad8 100644
--- a/lms/djangoapps/django_comment_client/utils.py
+++ b/lms/djangoapps/django_comment_client/utils.py
@@ -43,7 +43,8 @@ def get_full_modules():
         class_path = settings.MODULESTORE['default']['ENGINE']
         module_path, _, class_name = class_path.rpartition('.')
         class_ = getattr(import_module(module_path), class_name)
-        modulestore = class_(**dict(settings.MODULESTORE['default']['OPTIONS'].items() + [('eager', True)]))
+        # TODO (vshnayder): wth is this doing???
+        modulestore = class_(**dict(settings.MODULESTORE['default']['OPTIONS'].items()))
         _FULLMODULES = modulestore.modules
     return _FULLMODULES
 
@@ -76,7 +77,7 @@ def initialize_discussion_info(request, course):
 
     _is_course_discussion = lambda x: x[0].dict()['category'] == 'discussion' \
                          and x[0].dict()['course'] == course_name
-    
+
     _get_module_descriptor = operator.itemgetter(1)
 
     def _get_module(module_descriptor):
@@ -135,8 +136,8 @@ class HtmlResponse(HttpResponse):
     def __init__(self, html=''):
         super(HtmlResponse, self).__init__(html, content_type='text/plain')
 
-class ViewNameMiddleware(object):  
-    def process_view(self, request, view_func, view_args, view_kwargs):  
+class ViewNameMiddleware(object):
+    def process_view(self, request, view_func, view_args, view_kwargs):
         request.view_name = view_func.__name__
 
 class QueryCountDebugMiddleware(object):
diff --git a/lms/envs/common.py b/lms/envs/common.py
index ce08bf9666..0e2b2d38e6 100644
--- a/lms/envs/common.py
+++ b/lms/envs/common.py
@@ -219,7 +219,6 @@ MODULESTORE = {
         'OPTIONS': {
             'data_dir': DATA_DIR,
             'default_class': 'xmodule.hidden_module.HiddenDescriptor',
-            'eager': True,
         }
     }
 }

From 27fe80d57cdec4be21300562456dcff75ccee271 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Andr=C3=A9s=20Rocha?= <rocha@edx.org>
Date: Fri, 31 Aug 2012 06:29:07 -0400
Subject: [PATCH 109/181] [34078525] Use system random to generate passwords

---
 common/djangoapps/external_auth/views.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py
index 4dacd99d28..99f14ffa2a 100644
--- a/common/djangoapps/external_auth/views.py
+++ b/common/djangoapps/external_auth/views.py
@@ -66,7 +66,8 @@ def default_render_failure(request,
 
 def generate_password(length=12, chars=string.letters + string.digits):
     """Generate internal password for externally authenticated user"""
-    return ''.join([random.choice(chars) for i in range(length)])
+    choice = random.SystemRandom().choice
+    return ''.join([choice(chars) for i in range(length)])
 
 
 @csrf_exempt

From d702082daaf69f6e179b0ddf79917d83f4e8fc38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Andr=C3=A9s=20Rocha?= <rocha@edx.org>
Date: Fri, 31 Aug 2012 06:31:18 -0400
Subject: [PATCH 110/181] [34078525] Add test settings for openid

---
 common/djangoapps/external_auth/views.py | 3 ++-
 lms/envs/dev.py                          | 2 +-
 lms/envs/test.py                         | 5 +++++
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py
index 99f14ffa2a..c5351f32e6 100644
--- a/common/djangoapps/external_auth/views.py
+++ b/common/djangoapps/external_auth/views.py
@@ -244,7 +244,8 @@ def ssl_login(request):
             # try the direct apache2 SSL key
             cert = request._req.subprocess_env.get(certkey, '')
         except Exception:
-            pass
+            cert = None
+
     if not cert:
         # no certificate information - go onward to main index
         return student_views.index(request)
diff --git a/lms/envs/dev.py b/lms/envs/dev.py
index e1d6071bbf..f36383951e 100644
--- a/lms/envs/dev.py
+++ b/lms/envs/dev.py
@@ -116,7 +116,7 @@ OPENID_UPDATE_DETAILS_FROM_SREG = True
 OPENID_SSO_SERVER_URL = 'https://www.google.com/accounts/o8/id'	# TODO: accept more endpoints
 OPENID_USE_AS_ADMIN_LOGIN = False
 
-OPENID_PROVIDER_TRUSTED_ROOTS = ['*'] 
+OPENID_PROVIDER_TRUSTED_ROOTS = ['*']
 
 ################################ MIT Certificates SSL Auth #################################
 MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True
diff --git a/lms/envs/test.py b/lms/envs/test.py
index c164889d79..7cab4cb52c 100644
--- a/lms/envs/test.py
+++ b/lms/envs/test.py
@@ -123,6 +123,11 @@ CACHES = {
 # Dummy secret key for dev
 SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'
 
+################################## OPENID ######################################
+MITX_FEATURES['AUTH_USE_OPENID'] = True
+MITX_FEATURES['AUTH_USE_OPENID_PROVIDER'] = True
+OPENID_PROVIDER_TRUSTED_ROOTS = ['*']
+
 ############################ FILE UPLOADS (ASKBOT) #############################
 DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
 MEDIA_ROOT = TEST_ROOT / "uploads"

From 0a81684ac308a2a85e3adf7f7c7df3dfefec5739 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Andr=C3=A9s=20Rocha?= <rocha@edx.org>
Date: Fri, 31 Aug 2012 06:33:39 -0400
Subject: [PATCH 111/181] [34078525] Fix error saving open_id request in
 session

---
 common/djangoapps/external_auth/views.py | 56 +++++++++++++-----------
 1 file changed, 30 insertions(+), 26 deletions(-)

diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py
index c5351f32e6..354f572f4d 100644
--- a/common/djangoapps/external_auth/views.py
+++ b/common/djangoapps/external_auth/views.py
@@ -267,26 +267,21 @@ def ssl_login(request):
 # -----------------------------------------------------------------------------
 
 
-def get_dict_for_openid(data):
-    """
-    Return a dictionary suitable for the OpenID library
-    """
-
-    return dict((k, v) for k, v in data.iteritems())
-
-
 def get_xrds_url(resource, request):
     """
     Return the XRDS url for a resource
     """
+    host = request.META['HTTP_HOST']
+
+    if not host.endswith('edx.org'):
+        return None
+
+    location = host + '/openid/provider/' + resource + '/'
 
-    location = request.META['HTTP_HOST'] + '/openid/provider/' + resource + '/'
     if request.is_secure():
-        url = 'https://' + location
+        return 'https://' + location
     else:
-        url = 'http://' + location
-
-    return url
+        return 'http://' + location
 
 
 def add_openid_simple_registration(request, response, data):
@@ -402,17 +397,24 @@ def provider_login(request):
     OpenID login endpoint
     """
 
-    # initialize store and server
+    # make and validate endpoint
     endpoint = get_xrds_url('login', request)
+    if not endpoint:
+        return default_render_failure(request, "Invalid OpenID request")
+
+    # initialize store and server
     store = FileOpenIDStore('/tmp/openid_provider')
     server = Server(store, endpoint)
 
     # handle OpenID request
-    query = get_dict_for_openid(request.REQUEST)
+    querydict = dict(request.REQUEST.items())
     error = False
     if 'openid.mode' in request.GET or 'openid.mode' in request.POST:
         # decode request
-        openid_request = server.decodeRequest(query)
+        openid_request = server.decodeRequest(querydict)
+
+        if not openid_request:
+            return default_render_failure(request, "Invalid OpenID request")
 
         # don't allow invalid and non-trusted trust roots
         if not validate_trust_root(openid_request):
@@ -427,7 +429,7 @@ def provider_login(request):
         elif openid_request.mode == 'checkid_setup':
             if openid_request.idSelect():
                 # remember request and original path
-                request.session['openid_request'] = {
+                request.session['openid_setup'] = {
                     'request': openid_request,
                     'url': request.get_full_path()
                 }
@@ -443,12 +445,14 @@ def provider_login(request):
                                     server.handleRequest(openid_request), {})
 
     # handle login
-    if request.method == 'POST' and 'openid_request' in request.session:
+    if request.method == 'POST' and 'openid_setup' in request.session:
         # get OpenID request from session
-        openid_request = request.session['openid_request']
-        del request.session['openid_request']
+        openid_setup = request.session['openid_setup']
+        openid_request = openid_setup['request']
+        openid_request_url = openid_setup['url']
+        del request.session['openid_setup']
 
-        # don't allow invalid and non-*.cs50.net trust roots
+        # don't allow invalid trust roots
         if not validate_trust_root(openid_request):
             return default_render_failure(request, "Invalid OpenID trust root")
 
@@ -460,7 +464,7 @@ def provider_login(request):
             request.session['openid_error'] = True
             msg = "OpenID login failed - Unknown user email: {0}".format(email)
             log.warning(msg)
-            return HttpResponseRedirect(openid_request['url'])
+            return HttpResponseRedirect(openid_request_url)
 
         # attempt to authenticate user
         username = user.username
@@ -471,7 +475,7 @@ def provider_login(request):
             msg = "OpenID login failed - password for {0} is invalid"
             msg = msg.format(email)
             log.warning(msg)
-            return HttpResponseRedirect(openid_request['url'])
+            return HttpResponseRedirect(openid_request_url)
 
         # authentication succeeded, so log user in
         if user is not None and user.is_active:
@@ -486,10 +490,10 @@ def provider_login(request):
 
             # redirect user to return_to location
             url = endpoint + urlquote(user.username)
-            response = openid_request['request'].answer(True, None, url)
+            response = openid_request.answer(True, None, url)
 
             return provider_respond(server,
-                                    openid_request['request'],
+                                    openid_request,
                                     response,
                                     {
                                         'fullname': profile.name,
@@ -499,7 +503,7 @@ def provider_login(request):
         request.session['openid_error'] = True
         msg = "Login failed - Account not active for user {0}".format(username)
         log.warning(msg)
-        return HttpResponseRedirect(openid_request['url'])
+        return HttpResponseRedirect(openid_request_url)
 
     # determine consumer domain if applicable
     return_to = ''

From 327b3a46489ec8fa2d661e4554b65e09edf4c935 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Andr=C3=A9s=20Rocha?= <rocha@edx.org>
Date: Fri, 31 Aug 2012 06:34:22 -0400
Subject: [PATCH 112/181] [34078525] Log openid root_trust validation failures
 as errors

---
 common/djangoapps/external_auth/views.py | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py
index 354f572f4d..5cf21ca68d 100644
--- a/common/djangoapps/external_auth/views.py
+++ b/common/djangoapps/external_auth/views.py
@@ -353,39 +353,36 @@ def validate_trust_root(openid_request):
 
     trusted_roots = getattr(settings, 'OPENID_PROVIDER_TRUSTED_ROOT', None)
 
-    if trusted_roots is None:
-        log.debug('not using trusted roots')
+    if not trusted_roots:
         # not using trusted roots
         return True
 
-    log.debug('validating trusted roots')
-
     # don't allow empty trust roots
     if (not hasattr(openid_request, 'trust_root') or
-            openid_request.trust_root is None):
-        log.debug('no trust_root')
+            not openid_request.trust_root):
+        log.error('no trust_root')
         return False
 
     # ensure trust root parses cleanly (one wildcard, of form *.foo.com, etc.)
     trust_root = TrustRoot.parse(openid_request.trust_root)
-    if trust_root is None:
-        log.debug('invalid trust_root')
+    if not trust_root:
+        log.error('invalid trust_root')
         return False
 
     # don't allow empty return tos
     if (not hasattr(openid_request, 'return_to') or
-            openid_request.return_to is None):
-        log.debug('empty return_to')
+            not openid_request.return_to):
+        log.error('empty return_to')
         return False
 
     # ensure return to is within trust root
     if not trust_root.validateURL(openid_request.return_to):
-        log.debug('invalid return_to')
+        log.error('invalid return_to')
         return False
 
     # check that the root matches the ones we trust
     if not any(r for r in trusted_roots if fnmatch.fnmatch(trust_root, r)):
-        log.debug('non-trusted root')
+        log.error('non-trusted root')
         return False
 
     return True

From a98dcd958e24d219c4ac0eb349c3ccdca688274e Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Fri, 31 Aug 2012 07:09:54 -0400
Subject: [PATCH 113/181] Let admins see answers of problems at any time

---
 common/lib/xmodule/xmodule/capa_module.py | 4 ++++
 common/lib/xmodule/xmodule/x_module.py    | 1 +
 2 files changed, 5 insertions(+)

diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py
index cfe3d2d48b..0d812eda76 100644
--- a/common/lib/xmodule/xmodule/capa_module.py
+++ b/common/lib/xmodule/xmodule/capa_module.py
@@ -347,6 +347,10 @@ class CapaModule(XModule):
         if self.show_answer == "never":
             return False
 
+        # Admins can see the answer, unless the problem explicitly prevents it
+        if self.system.user_is_staff:
+            return True
+
         if self.show_answer == 'attempted':
             return self.attempts > 0
 
diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py
index 0dc16bd976..b2d883b38d 100644
--- a/common/lib/xmodule/xmodule/x_module.py
+++ b/common/lib/xmodule/xmodule/x_module.py
@@ -765,6 +765,7 @@ class ModuleSystem(object):
         self.replace_urls = replace_urls
         self.node_path = node_path
         self.anonymous_student_id = anonymous_student_id
+        self.user_is_staff = user.is_staff
 
     def get(self, attr):
         '''	provide uniform access to attributes (like etree).'''

From 32cb0897e9b95d3abdcd9419ad30859ee226c2f4 Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Fri, 31 Aug 2012 04:24:52 -0700
Subject: [PATCH 114/181] Fixing sequentials nested inside of verticals,
 provides a path to sequentials nested in sequentials. Changes jump_to to
 redirect to courseware index.

---
 common/lib/xmodule/xmodule/modulestore/search.py | 15 +++++++++------
 lms/djangoapps/courseware/views.py               |  7 +++++--
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/search.py b/common/lib/xmodule/xmodule/modulestore/search.py
index 87f1cbf825..1eb880c56a 100644
--- a/common/lib/xmodule/xmodule/modulestore/search.py
+++ b/common/lib/xmodule/xmodule/modulestore/search.py
@@ -93,11 +93,14 @@ def path_to_location(modulestore, course_id, location):
     # Figure out the position
     position = None
     if n > 3:
-        section_desc = modulestore.get_instance(course_id, path[2])
-        child_locs = [c.location for c in section_desc.get_children()]
-        # positions are 1-indexed, and should be strings to be consistent with
-        # url parsing.
-        position = str(child_locs.index(path[3]) + 1)
-
+        position_list = []
+        for path_index in range(2, n-1):
+            if path[path_index].category == 'sequential':
+                section_desc = modulestore.get_instance(course_id, path[path_index])
+                child_locs = [c.location for c in section_desc.get_children()]
+                # positions are 1-indexed, and should be strings to be consistent with
+                # url parsing.
+                position_list.append(str(child_locs.index(path[path_index+1]) + 1))
+        position = "_".join(position_list)
 
     return (course_id, chapter, section, position)
diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py
index 3f0f54147b..d1a52f06c1 100644
--- a/lms/djangoapps/courseware/views.py
+++ b/lms/djangoapps/courseware/views.py
@@ -218,8 +218,11 @@ def jump_to(request, course_id, location):
         raise Http404("This location is not in any class: {0}".format(location))
 
     # Rely on index to do all error handling and access control.
-    return index(request, course_id, chapter, section, position)
-
+    return redirect('courseware_position',
+                    course_id=course_id, 
+                    chapter=chapter, 
+                    section=section, 
+                    position=position)
 @ensure_csrf_cookie
 def course_info(request, course_id):
     """

From 710898532568026312b803a95851ff4312257ff2 Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Fri, 31 Aug 2012 04:59:12 -0700
Subject: [PATCH 115/181] Adding context to posts in the forum view.

---
 .../django_comment_client/forum/views.py      |  11 +-
 lms/djangoapps/django_comment_client/utils.py | 112 +++++++++++-------
 lms/static/sass/_discussion.scss              |   5 +
 .../discussion/mustache/_content.mustache     |   5 +
 4 files changed, 88 insertions(+), 45 deletions(-)

diff --git a/lms/djangoapps/django_comment_client/forum/views.py b/lms/djangoapps/django_comment_client/forum/views.py
index 62055f0ab7..3d6515dfac 100644
--- a/lms/djangoapps/django_comment_client/forum/views.py
+++ b/lms/djangoapps/django_comment_client/forum/views.py
@@ -13,7 +13,7 @@ from courseware.access import has_access
 from urllib import urlencode
 from operator import methodcaller
 from django_comment_client.permissions import check_permissions_by_view
-from django_comment_client.utils import merge_dict, extract, strip_none, strip_blank
+from django_comment_client.utils import merge_dict, extract, strip_none, strip_blank, get_courseware_context
 
 import json
 import django_comment_client.utils as utils
@@ -99,6 +99,8 @@ def render_user_discussion(*args, **kwargs):
 
 def get_threads(request, course_id, discussion_id=None):
 
+    course = get_course_with_access(request.user, course_id, 'load')
+
     default_query_params = {
         'page': 1,
         'per_page': THREADS_PER_PAGE,
@@ -115,6 +117,13 @@ def get_threads(request, course_id, discussion_id=None):
 
     threads, page, num_pages = cc.Thread.search(query_params)
 
+    for thread in threads:
+        courseware_context = get_courseware_context(thread, course)
+        if courseware_context:
+            thread['courseware_location']  = courseware_context['courseware_location']
+            thread['courseware_title']  = courseware_context['courseware_title']
+
+
     query_params['page'] = page
     query_params['num_pages'] = num_pages
 
diff --git a/lms/djangoapps/django_comment_client/utils.py b/lms/djangoapps/django_comment_client/utils.py
index 516344d79b..e5b32d1d9c 100644
--- a/lms/djangoapps/django_comment_client/utils.py
+++ b/lms/djangoapps/django_comment_client/utils.py
@@ -1,3 +1,4 @@
+from collections import defaultdict 
 from importlib import import_module
 from courseware.models import StudentModuleCache
 from courseware.module_render import get_module
@@ -40,80 +41,92 @@ def merge_dict(dic1, dic2):
 def get_full_modules():
     global _FULLMODULES
     if not _FULLMODULES:
-        class_path = settings.MODULESTORE['default']['ENGINE']
-        module_path, _, class_name = class_path.rpartition('.')
-        class_ = getattr(import_module(module_path), class_name)
-        modulestore = class_(**dict(settings.MODULESTORE['default']['OPTIONS'].items() + [('eager', True)]))
-        _FULLMODULES = modulestore.modules
+        _FULLMODULES = modulestore().modules
     return _FULLMODULES
 
-def get_categorized_discussion_info(request, course):
+def get_discussion_id_map(course):
     """
         return a dict of the form {category: modules}
     """
     global _DISCUSSIONINFO
     if not _DISCUSSIONINFO:
-        initialize_discussion_info(request, course)
-    return _DISCUSSIONINFO['categorized']
+        initialize_discussion_info(course)
+    return _DISCUSSIONINFO['id_map']
 
 def get_discussion_title(request, course, discussion_id):
     global _DISCUSSIONINFO
     if not _DISCUSSIONINFO:
-        initialize_discussion_info(request, course)
-    title = _DISCUSSIONINFO['by_id'].get(discussion_id, {}).get('title', '(no title)')
+        initialize_discussion_info(course)
+    title = _DISCUSSIONINFO['id_map'].get(discussion_id, {}).get('title', '(no title)')
     return title
 
-def initialize_discussion_info(request, course):
+def get_discussion_category_map(course):
+
+    global _DISCUSSIONINFO
+    if not _DISCUSSIONINFO:
+        initialize_discussion_info(course)
+    return _DISCUSSIONINFO['category_map']
+
+def initialize_discussion_info(course):
 
     global _DISCUSSIONINFO
     if _DISCUSSIONINFO:
         return
 
     course_id = course.id
-    _, course_name, _ = course_id.split('/')
-    user = request.user
     url_course_id = course_id.replace('/', '_').replace('.', '_')
 
-    _is_course_discussion = lambda x: x[0].dict()['category'] == 'discussion' \
-                         and x[0].dict()['course'] == course_name
-    
-    _get_module_descriptor = operator.itemgetter(1)
+    all_modules = get_full_modules()[course_id]
 
-    def _get_module(module_descriptor):
-        print module_descriptor
-        module = get_module(user, request, module_descriptor.location, student_module_cache)
-        return module
+    discussion_id_map = {}
+    unexpanded_category_map = defaultdict(list)
+    for location, module in all_modules.items():
+        if location.category == 'discussion':
+            id = module.metadata['id']
+            category = module.metadata['discussion_category']
+            title = module.metadata['for']
+            sort_key = module.metadata.get('sort_key', title)
+            discussion_id_map[id] = {"location": location, "title": title}
+            category = " / ".join([x.strip() for x in category.split("/")])
+            unexpanded_category_map[category].append({"title": title, "id": id,
+                "sort_key": sort_key})
 
-    def _extract_info(module):
-        return {
-            'title': module.title,
-            'discussion_id': module.discussion_id,
-            'category': module.discussion_category,
-        }
+    category_map = {"entries": defaultdict(dict), "subcategories": defaultdict(dict)} 
+    for category_path, entries in unexpanded_category_map.items():
+        node = category_map["subcategories"]
+        path = [x.strip() for x in category_path.split("/")]
+        for level in path[:-1]:
+            if level not in node:
+                node[level] = {"subcategories": defaultdict(dict), 
+                               "entries": defaultdict(dict),
+                               "sort_key": level} 
+            node = node[level]["subcategories"]
 
-    def _pack_with_id(info):
-        return (info['discussion_id'], info)
+        level = path[-1]
+        if level not in node:
+            node[level] = {"subcategories": defaultdict(dict), 
+                            "entries": defaultdict(dict), 
+                            "sort_key": level}
+        for entry in entries:
+            node[level]["entries"][entry["title"]] = {"id": entry["id"], 
+                                                      "sort_key": entry["sort_key"]}
 
-    discussion_module_descriptors = map(_get_module_descriptor,
-                                        filter(_is_course_discussion,
-                                               get_full_modules().items()))
+    def sort_map_entries(map):
+        things = []
+        for title, entry in map["entries"].items():
+            things.append((title, entry))
+        for title, category in map["subcategories"].items():
+            things.append((title, category))
+            sort_map_entries(map["subcategories"][title])
+        map["children"] = [x[0] for x in sorted(things, key=lambda x: x[1]["sort_key"])]
 
-    student_module_cache = StudentModuleCache.cache_for_descriptor_descendents(user, course)
-
-    discussion_info = map(_extract_info, map(_get_module, discussion_module_descriptors))
+    sort_map_entries(category_map)
 
     _DISCUSSIONINFO = {}
 
-    _DISCUSSIONINFO['by_id'] = dict(map(_pack_with_id, discussion_info))
+    _DISCUSSIONINFO['id_map'] = discussion_id_map
 
-    _DISCUSSIONINFO['categorized'] = dict((category, list(l)) \
-                for category, l in itertools.groupby(discussion_info, operator.itemgetter('category')))
-
-    _DISCUSSIONINFO['categorized']['General'] = [{
-        'title': 'General discussion',
-        'discussion_id': url_course_id,
-        'category': 'General',
-    }]
+    _DISCUSSIONINFO['category_map'] = category_map
 
 class JsonResponse(HttpResponse):
     def __init__(self, data=None):
@@ -220,6 +233,17 @@ def extend_content(content):
     }
     return merge_dict(content, content_info)
 
+def get_courseware_context(content, course):
+    id_map = get_discussion_id_map(course)
+    id = content['commentable_id'] 
+    content_info = None
+    if id in id_map:
+        location = id_map[id]["location"].url()
+        title = id_map[id]["title"]
+        content_info = { "courseware_location": location, "courseware_title": title}
+    return content_info
+
+
 def safe_content(content):
     fields = [
         'id', 'title', 'body', 'course_id', 'anonymous', 'endorsed',
diff --git a/lms/static/sass/_discussion.scss b/lms/static/sass/_discussion.scss
index 6c1988684e..755ef91289 100644
--- a/lms/static/sass/_discussion.scss
+++ b/lms/static/sass/_discussion.scss
@@ -342,6 +342,11 @@ $tag-text-color: #5b614f;
       display: inline-block;
     }
 
+    .context{
+        margin-top: 1em;
+        font-size: $comment-font-size;
+    }
+
     .info {
       @include discussion-font;
       color: gray;
diff --git a/lms/templates/discussion/mustache/_content.mustache b/lms/templates/discussion/mustache/_content.mustache
index b4f3176931..88f53a23ac 100644
--- a/lms/templates/discussion/mustache/_content.mustache
+++ b/lms/templates/discussion/mustache/_content.mustache
@@ -27,6 +27,11 @@
             {{/content.tags}}
           </div>
         {{/thread}}
+        <div class="context">
+            {{#content.courseware_location}}
+               (this post is about <a href="../../jump_to/{{content.courseware_location}}">{{content.courseware_title}}</a>)
+            {{/content.courseware_location}}
+        </div>
         <div class="info">
           <div class="comment-time">
             <span class="timeago" title="{{content.updated_at}}">sometime</span> by

From 3f58e1dd01273f8235ecb8de81093dfe8f2c5034 Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Fri, 31 Aug 2012 05:07:41 -0700
Subject: [PATCH 116/181] Remove context from inline posts

---
 .../django_comment_client/forum/views.py      | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/lms/djangoapps/django_comment_client/forum/views.py b/lms/djangoapps/django_comment_client/forum/views.py
index 3d6515dfac..5f94eade77 100644
--- a/lms/djangoapps/django_comment_client/forum/views.py
+++ b/lms/djangoapps/django_comment_client/forum/views.py
@@ -71,6 +71,16 @@ def render_discussion(request, course_id, threads, *args, **kwargs):
 
     annotated_content_info = reduce(merge_dict, map(infogetter, threads), {})
 
+    if discussion_type != 'inline':
+        course = get_course_with_access(request.user, course_id, 'load')
+
+        for thread in threads:
+            courseware_context = get_courseware_context(thread, course)
+            if courseware_context:
+                thread['courseware_location']  = courseware_context['courseware_location']
+                thread['courseware_title']  = courseware_context['courseware_title']
+
+
     context = {
         'threads': threads,
         'discussion_id': discussion_id,
@@ -99,8 +109,6 @@ def render_user_discussion(*args, **kwargs):
 
 def get_threads(request, course_id, discussion_id=None):
 
-    course = get_course_with_access(request.user, course_id, 'load')
-
     default_query_params = {
         'page': 1,
         'per_page': THREADS_PER_PAGE,
@@ -117,13 +125,6 @@ def get_threads(request, course_id, discussion_id=None):
 
     threads, page, num_pages = cc.Thread.search(query_params)
 
-    for thread in threads:
-        courseware_context = get_courseware_context(thread, course)
-        if courseware_context:
-            thread['courseware_location']  = courseware_context['courseware_location']
-            thread['courseware_title']  = courseware_context['courseware_title']
-
-
     query_params['page'] = page
     query_params['num_pages'] = num_pages
 

From b58436545a5cea8fea816677a8ced1bab3712d66 Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Fri, 31 Aug 2012 06:45:59 -0700
Subject: [PATCH 117/181] Add note about positional modules; check for
 videosequence.

---
 common/lib/xmodule/xmodule/modulestore/search.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/search.py b/common/lib/xmodule/xmodule/modulestore/search.py
index 1eb880c56a..f9901e8bfe 100644
--- a/common/lib/xmodule/xmodule/modulestore/search.py
+++ b/common/lib/xmodule/xmodule/modulestore/search.py
@@ -92,10 +92,18 @@ def path_to_location(modulestore, course_id, location):
     section = path[2].name if n > 2 else None
     # Figure out the position
     position = None
+
+    # This block of code will find the position of a module within a nested tree
+    # of modules. If a problem is on tab 2 of a sequence that's on tab 3 of a
+    # sequence, the resulting position is 3_2. However, no positional modules
+    # (e.g. sequential and videosequence) currently deal with this form of
+    # representing nested positions. This needs to happen before jumping to a
+    # module nested in more than one positional module will work.
     if n > 3:
         position_list = []
         for path_index in range(2, n-1):
-            if path[path_index].category == 'sequential':
+            category = path[path_index].category
+            if  category == 'sequential' or category == 'videosequence':
                 section_desc = modulestore.get_instance(course_id, path[path_index])
                 child_locs = [c.location for c in section_desc.get_children()]
                 # positions are 1-indexed, and should be strings to be consistent with

From e6f9f8d9b0b6fc6b8a3b7d8f0abd8296b225ff6c Mon Sep 17 00:00:00 2001
From: Kyle Fiedler <kfiedler@thoughtbot.com>
Date: Fri, 31 Aug 2012 11:15:06 -0400
Subject: [PATCH 118/181] Add js to the problem and move around that css and js
 to accomodate Tonys new HTML Structure

---
 .../lib/xmodule/xmodule/css/capa/display.scss | 36 +++++++++----------
 lms/templates/problem.html                    | 24 +++++++++++++
 2 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/common/lib/xmodule/xmodule/css/capa/display.scss b/common/lib/xmodule/xmodule/css/capa/display.scss
index e171d35513..a368167448 100644
--- a/common/lib/xmodule/xmodule/css/capa/display.scss
+++ b/common/lib/xmodule/xmodule/css/capa/display.scss
@@ -453,37 +453,33 @@ section.problem {
       }
     }
 
-    .longform, .shortform {
+    > section {
       border: 1px solid #ddd;
-      padding: 9px;
+      padding: 9px 9px 20px;
       margin-bottom: 10px;
       background: #FFF;
+      position: relative;
       @include box-shadow(inset 0 0 0 1px #eee);
       @include border-radius(3px);
 
       p:last-of-type {
         margin-bottom: 0;
       }
-    }
 
-    a.full {
-      @include position(absolute,  0 0 1px 0px);
-      font-size: .8em;
-      padding: 4px;
-      text-align: right;
-      width: 100%;
-      display: block;
-      background: #F3F3F3;
-      @include box-sizing(border-box);
-    }
+      .shortform {
+        margin-bottom: .6em;
+      }
 
-    .longform {
-      position: relative;
-      padding-bottom: 20px;
-    }
-
-    .longform-header {
-      margin-bottom: .6em;
+      a.full {
+        @include position(absolute,  0 0 1px 0px);
+        font-size: .8em;
+        padding: 4px;
+        text-align: right;
+        width: 100%;
+        display: block;
+        background: #F3F3F3;
+        @include box-sizing(border-box);
+      }
     }
   }
 }
diff --git a/lms/templates/problem.html b/lms/templates/problem.html
index ed49b3bd5d..65b8193df9 100644
--- a/lms/templates/problem.html
+++ b/lms/templates/problem.html
@@ -32,3 +32,27 @@
   </section>
 </section>
 
+<%block name="js_extra">
+<script type="text/javascript" charset="utf-8">
+  $(function(){
+      // this should be brought back into problems
+      $('.longform').hide();
+      $('.shortform').append('<a href="#" class="full">See full output</a>');
+
+      $('.full').click(function() {
+        $(this).parent().siblings().slideToggle();
+        $(this).parent().parent().toggleClass('open');
+        var text = $(this).text() == 'See full output' ? 'Hide output' : 'See full output';
+        $(this).text(text);
+        return false;
+      });
+
+      $('.collapsible section').hide()
+      $('.collapsible header a').click(function() {
+        $(this).parent().siblings().slideToggle();
+        $(this).parent().parent().toggleClass('open');
+        return false
+      });
+    });
+</script>
+</%block>

From 440539c081b1686aaeca6855d043e97e1c9295b4 Mon Sep 17 00:00:00 2001
From: Kyle Fiedler <kfiedler@thoughtbot.com>
Date: Fri, 31 Aug 2012 13:27:32 -0400
Subject: [PATCH 119/181] Removed js from textbox

---
 common/lib/capa/capa/templates/textbox.html | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/common/lib/capa/capa/templates/textbox.html b/common/lib/capa/capa/templates/textbox.html
index 7800e6031e..271d7795e0 100644
--- a/common/lib/capa/capa/templates/textbox.html
+++ b/common/lib/capa/capa/templates/textbox.html
@@ -45,25 +45,6 @@
         indentWithTabs: true,
         smartIndent: false
       });
-
-      // this should be brought back into problems
-      $('.longform-body').hide();
-      $('.longform-header').append('<a href="" class="full">See full output</a>');
-
-      $('.full').click(function() {
-        $(this).parent().siblings().slideToggle();
-        $(this).parent().parent().toggleClass('open');
-        var text = $(this).text() == 'See full output' ? 'Hide output' : 'See full output';
-        $(this).text(text);
-        return false;
-      });
-
-      $('.collapsible section').hide()
-      $('.collapsible header a').click(function() {
-        $(this).parent().siblings().slideToggle();
-        $(this).parent().parent().toggleClass('open');
-        return false
-      });
     });
   </script>
 </section>

From 520256f67a37f5274c8ffefce70882cbb1e53f14 Mon Sep 17 00:00:00 2001
From: Kyle Fiedler <kfiedler@thoughtbot.com>
Date: Fri, 31 Aug 2012 14:17:03 -0400
Subject: [PATCH 120/181] Added fix for the sequnce nave hovers

---
 common/lib/xmodule/xmodule/css/sequence/display.scss | 1 -
 1 file changed, 1 deletion(-)

diff --git a/common/lib/xmodule/xmodule/css/sequence/display.scss b/common/lib/xmodule/xmodule/css/sequence/display.scss
index 7b8dc5f57c..25d2c26dda 100644
--- a/common/lib/xmodule/xmodule/css/sequence/display.scss
+++ b/common/lib/xmodule/xmodule/css/sequence/display.scss
@@ -37,7 +37,6 @@ nav.sequence-nav {
     height: 44px;
     margin: 0 30px;
     @include linear-gradient(top, #ddd, #eee);
-    overflow: hidden;
     @include box-shadow(0 1px 3px rgba(0, 0, 0, .1) inset);
   }
 

From 262b58778f3d825d916290a031b06aec4b7e8053 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Fri, 31 Aug 2012 14:37:31 -0400
Subject: [PATCH 121/181] Accordion distinguishes items to be graded

---
 lms/djangoapps/courseware/module_render.py | 4 +++-
 lms/templates/accordion.html               | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py
index 82ec3cadeb..319ec25f43 100644
--- a/lms/djangoapps/courseware/module_render.py
+++ b/lms/djangoapps/courseware/module_render.py
@@ -98,7 +98,9 @@ def toc_for_course(user, request, course, active_chapter, active_section, course
                                  'url_name': section.url_name,
                                  'format': section.metadata.get('format', ''),
                                  'due': section.metadata.get('due', ''),
-                                 'active': active})
+                                 'active': active,
+                                 'graded': section.metadata.get('graded', False),
+                                 })
 
         chapters.append({'display_name': chapter.display_name,
                          'url_name': chapter.url_name,
diff --git a/lms/templates/accordion.html b/lms/templates/accordion.html
index 0d3dee9495..fa91927462 100644
--- a/lms/templates/accordion.html
+++ b/lms/templates/accordion.html
@@ -7,7 +7,7 @@
 
     <ul>
       % for section in chapter['sections']:
-          <li${' class="active"' if 'active' in section and section['active'] else ''}>
+          <li class="${'active' if 'active' in section and section['active'] else ''} ${'graded'  if 'graded' in section and section['graded'] else ''}">
             <a href="${reverse('courseware_section', args=[course_id, chapter['url_name'], section['url_name']])}">
               <p>${section['display_name']}
               <span class="subtitle">

From 77612b0c602e8a0e4195fea1847a0062023e7390 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Fri, 31 Aug 2012 14:38:39 -0400
Subject: [PATCH 122/181] Adjust comments

---
 lms/djangoapps/courseware/module_render.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py
index 319ec25f43..59414274a6 100644
--- a/lms/djangoapps/courseware/module_render.py
+++ b/lms/djangoapps/courseware/module_render.py
@@ -62,7 +62,7 @@ def toc_for_course(user, request, course, active_chapter, active_section, course
 
     where SECTIONS is a list
     [ {'display_name': name, 'url_name': url_name,
-       'format': format, 'due': due, 'active' : bool}, ...]
+       'format': format, 'due': due, 'active' : bool, 'graded': bool}, ...]
 
     active is set for the section and chapter corresponding to the passed
     parameters, which are expected to be url_names of the chapter+section.

From 29be6f8209515ff173510301cc8b5453ae440cbd Mon Sep 17 00:00:00 2001
From: David Ormsbee <dave@edx.org>
Date: Fri, 31 Aug 2012 15:16:31 -0400
Subject: [PATCH 123/181] Prefetch user groups to speed up auth checks required
 to render Progress page view for students

---
 lms/djangoapps/courseware/access.py | 2 +-
 lms/djangoapps/courseware/views.py  | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/lms/djangoapps/courseware/access.py b/lms/djangoapps/courseware/access.py
index dbe4ff376d..91c769f90a 100644
--- a/lms/djangoapps/courseware/access.py
+++ b/lms/djangoapps/courseware/access.py
@@ -321,7 +321,7 @@ def _has_staff_access_to_location(user, location):
         return True
 
     # If not global staff, is the user in the Auth group for this class?
-    user_groups = [x[1] for x in user.groups.values_list()]
+    user_groups = [g.name for g in user.groups.all()]
     staff_group = _course_staff_group_name(location)
     if staff_group in user_groups:
         debug("Allow: user in group %s", staff_group)
diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py
index 92f6716320..dab04a0957 100644
--- a/lms/djangoapps/courseware/views.py
+++ b/lms/djangoapps/courseware/views.py
@@ -327,6 +327,7 @@ def progress(request, course_id, student_id=None):
 
     # NOTE: To make sure impersonation by instructor works, use
     # student instead of request.user in the rest of the function.
+    student = User.objects.prefetch_related("groups").get(id=student.id)
 
     student_module_cache = StudentModuleCache.cache_for_descriptor_descendents(
         course_id, student, course)

From 04439a321791ef58a8e2627a185ea8fc91775c3f Mon Sep 17 00:00:00 2001
From: David Ormsbee <dave@edx.org>
Date: Fri, 31 Aug 2012 15:20:13 -0400
Subject: [PATCH 124/181] add comments explaining prefetch need

---
 lms/djangoapps/courseware/views.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py
index dab04a0957..6b77792203 100644
--- a/lms/djangoapps/courseware/views.py
+++ b/lms/djangoapps/courseware/views.py
@@ -327,6 +327,9 @@ def progress(request, course_id, student_id=None):
 
     # NOTE: To make sure impersonation by instructor works, use
     # student instead of request.user in the rest of the function.
+
+    # The pre-fetching of groups is done to make auth checks not require an 
+    # additional DB lookup (this kills the Progress page in particular).
     student = User.objects.prefetch_related("groups").get(id=student.id)
 
     student_module_cache = StudentModuleCache.cache_for_descriptor_descendents(

From 0ff70472c936f9ecfb2064a85447d6657e53e636 Mon Sep 17 00:00:00 2001
From: Kyle Fiedler <kfiedler@thoughtbot.com>
Date: Thu, 30 Aug 2012 11:08:07 -0400
Subject: [PATCH 125/181] Added styles for dashboard to have link around whole
 class

---
 lms/static/sass/multicourse/_dashboard.scss | 28 +++++++++----
 lms/templates/dashboard.html                | 46 +++++++++++----------
 2 files changed, 44 insertions(+), 30 deletions(-)

diff --git a/lms/static/sass/multicourse/_dashboard.scss b/lms/static/sass/multicourse/_dashboard.scss
index f37c772aef..002f35cf68 100644
--- a/lms/static/sass/multicourse/_dashboard.scss
+++ b/lms/static/sass/multicourse/_dashboard.scss
@@ -261,7 +261,7 @@
           padding: 12px 0px;
           width: 100%;
 
-          a.university {
+          .university {
             background: rgba(255,255,255, 1);
             border: 1px solid rgb(180,180,180);
             @include border-radius(3px);
@@ -269,17 +269,14 @@
             color: $lighter-base-font-color;
             display: block;
             font-style: italic;
+            font-family: $sans-serif;
+            font-size: 16px;
             font-weight: 800;
             @include inline-block;
             margin-right: 10px;
+            margin-bottom: 0;
             padding: 5px 10px;
-
             float: left;
-
-            &:hover {
-              color: $blue;
-              text-decoration: none;
-            }
           }
 
           h3 {
@@ -392,18 +389,30 @@
         }
       }
 
-      &:hover {
+      > a:hover {
         .cover {
           .shade {
             background: rgba(255,255,255, 0.1);
             @include background-image(linear-gradient(-90deg, rgba(255,255,255, 0.3) 0%,
-                                                              rgba(0,0,0, 0.3) 100%));
+            rgba(0,0,0, 0.3) 100%));
           }
 
           .arrow {
             opacity: 1;
           }
         }
+
+        .info {
+          background: darken(rgb(250,250,250), 5%);
+          @include background-image(linear-gradient(-90deg, darken(rgb(253,253,253), 3%), darken(rgb(240,240,240), 5%)));
+          border-color: darken(rgb(190,190,190), 10%);
+
+          .course-status {
+            background: darken($yellow, 3%);
+            border-color: darken(rgb(200,200,200), 3%);
+            @include box-shadow(0 1px 0 0 rgba(255,255,255, 0.6));
+          }
+        }
       }
     }
 
@@ -420,5 +429,6 @@
         color: #333;
       }
     }
+
   }
 }
diff --git a/lms/templates/dashboard.html b/lms/templates/dashboard.html
index 770e849841..b73e3a57bc 100644
--- a/lms/templates/dashboard.html
+++ b/lms/templates/dashboard.html
@@ -72,30 +72,34 @@
           else:
             course_target = reverse('about_course', args=[course.id])
           %>
-          <a href="${course_target}" class="cover" style="background-image: url('${course_image_url(course)}')">
-            <div class="shade"></div>
-            <div class="arrow">&#10095;</div>
-          </a>
-          <section class="info">
-            <hgroup>
-              <a href="${reverse('university_profile', args=[course.org])}" class="university">${get_course_about_section(course, 'university')}</a>
-              <h3><a href="${course_target}">${course.number} ${course.title}</a></h3>
-            </hgroup>
-            <section class="course-status">
-              <p>Class Starts - <span>${course.start_date_text}</span></div>
+
+          <a href="${course_target}">
+            <section class="cover" style="background-image: url('${course_image_url(course)}')">
+              <div class="shade"></div>
+              <div class="arrow">&#10095;</div>
             </section>
-            <section class="meta">
-              <div class="course-work-icon"></div>
-              <div class="progress">
-                <div class="meter">
-                  <div class="meter-fill"></div>
+
+            <section class="info">
+              <hgroup>
+                <h2 class="university">${get_course_about_section(course, 'university')}</h2>
+                <h3>${course.number} ${course.title}</h3>
+              </hgroup>
+              <section class="course-status">
+                <p>Class Starts - <span>${course.start_date_text}</span></div>
+              </section>
+              <section class="meta">
+                <div class="course-work-icon"></div>
+                <div class="progress">
+                  <div class="meter">
+                    <div class="meter-fill"></div>
+                  </div>
                 </div>
-              </div>
-              <div class="complete">
-                ##<p><span class="completeness">60%</span> complete</p>
-              </div>
+                <div class="complete">
+                  ##<p><span class="completeness">60%</span> complete</p>
+                </div>
+              </section>
             </section>
-          </section>
+          </a>
         </article>
         <a href="#unenroll-modal" class="unenroll" rel="leanModal" data-course-id="${course.id}" data-course-number="${course.number}">Unregister</a>
 

From 7c1cc4ed219b3fab9409eea6e599f043d5d1aec1 Mon Sep 17 00:00:00 2001
From: Kyle Fiedler <kfiedler@thoughtbot.com>
Date: Thu, 30 Aug 2012 11:47:40 -0400
Subject: [PATCH 126/181] Added hover on course about and more actionable text

---
 lms/static/sass/multicourse/_course_about.scss | 14 ++++++++++++++
 lms/templates/portal/course_about.html         |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/lms/static/sass/multicourse/_course_about.scss b/lms/static/sass/multicourse/_course_about.scss
index 92d19dff86..55996d8da9 100644
--- a/lms/static/sass/multicourse/_course_about.scss
+++ b/lms/static/sass/multicourse/_course_about.scss
@@ -115,6 +115,12 @@
             }
           }
 
+          a {
+            &:hover, &:visited {
+              text-decoration: none;
+            }
+          }
+
           span.register {
             background: lighten($blue, 20%);
             border: 1px solid $blue;
@@ -126,6 +132,14 @@
             text-transform: uppercase;
             text-align: center;
             width: flex-grid(12);
+            @include transition();
+
+            &:hover {
+              background: lighten($pink, 20%);
+              border: 1px solid $pink;
+              color: darken($pink, 20%);
+              text-decoration: none;
+            }
           }
         }
       }
diff --git a/lms/templates/portal/course_about.html b/lms/templates/portal/course_about.html
index 4931f1fed6..a7e7abad20 100644
--- a/lms/templates/portal/course_about.html
+++ b/lms/templates/portal/course_about.html
@@ -74,7 +74,7 @@
 	        %if show_link:
                 <a href="${course_target}">
                 %endif
-                <span class="register disabled">You are registered for this course (${course.number}).</span>
+                <span class="register disabled">You are registered for this course (${course.number}). <strong>View Courseware.<strong></span>
 	        %if show_link:
                 </a>
                 %endif

From 3edf1a6a84e5eab2f08b3ade07520aa811cc96ac Mon Sep 17 00:00:00 2001
From: Kyle Fiedler <kfiedler@thoughtbot.com>
Date: Fri, 31 Aug 2012 15:58:31 -0400
Subject: [PATCH 127/181] Added buttons for both pages to make it even more
 clear to start class

---
 .../sass/multicourse/_course_about.scss       | 29 ++++--
 lms/static/sass/multicourse/_dashboard.scss   | 91 ++++---------------
 lms/templates/dashboard.html                  | 14 +--
 lms/templates/portal/course_about.html        |  2 +-
 4 files changed, 42 insertions(+), 94 deletions(-)

diff --git a/lms/static/sass/multicourse/_course_about.scss b/lms/static/sass/multicourse/_course_about.scss
index 55996d8da9..3f2813ba0e 100644
--- a/lms/static/sass/multicourse/_course_about.scss
+++ b/lms/static/sass/multicourse/_course_about.scss
@@ -121,6 +121,24 @@
             }
           }
 
+          strong {
+            @include button(shiny, $blue);
+            @include box-sizing(border-box);
+            @include border-radius(3px);
+            display: block;
+            float: left;
+            font: normal 1.2rem/1.6rem $sans-serif;
+            letter-spacing: 1px;
+            padding: 10px 0px;
+            text-transform: uppercase;
+            text-align: center;
+            width: flex-grid(3, 8);
+
+            &:hover {
+              color: rgb(255,255,255);
+            }
+          }
+
           span.register {
             background: lighten($blue, 20%);
             border: 1px solid $blue;
@@ -131,15 +149,10 @@
             padding: 10px 0px 8px;
             text-transform: uppercase;
             text-align: center;
-            width: flex-grid(12);
+            float: left;
+            margin: 1px flex-gutter(8) 0 0;
             @include transition();
-
-            &:hover {
-              background: lighten($pink, 20%);
-              border: 1px solid $pink;
-              color: darken($pink, 20%);
-              text-decoration: none;
-            }
+            width: flex-grid(5, 8);
           }
         }
       }
diff --git a/lms/static/sass/multicourse/_dashboard.scss b/lms/static/sass/multicourse/_dashboard.scss
index 002f35cf68..53418bc0dd 100644
--- a/lms/static/sass/multicourse/_dashboard.scss
+++ b/lms/static/sass/multicourse/_dashboard.scss
@@ -303,8 +303,12 @@
           background: $yellow;
           border: 1px solid rgb(200,200,200);
           @include box-shadow(0 1px 0 0 rgba(255,255,255, 0.6));
-          margin-top: 16px;
+          margin-top: 17px;
+          margin-right: flex-gutter();
           padding: 5px;
+          width: flex-grid(8);
+          float: left;
+          @include box-sizing(border-box);
 
           p {
             color: $lighter-base-font-color;
@@ -314,78 +318,19 @@
           }
         }
 
-        .meta {
-          @include clearfix;
-          margin-top: 22px;
-          position: relative;
-          @include transition(opacity, 0.15s, linear);
-          width: 100%;
-
-
-          .course-work-icon {
-            @include background-image(url('../images/portal-icons/pencil-icon.png'));
-            background-size: cover;
-            float: left;
-            height: 22px;
-            opacity: 0.7;
-            width: 22px;
-          }
-
-          .complete {
-            float: right;
-
-            p {
-              color: $lighter-base-font-color;
-              font-style: italic;
-              @include inline-block;
-              text-align: right;
-              text-shadow: 0 1px rgba(255,255,255, 0.6);
-
-              .completeness {
-                color: $base-font-color;
-                font-weight: 700;
-                margin-right: 5px;
-              }
-            }
-          }
-
-          .progress {
-            @include box-shadow(0 1px 0 0 rgba(255,255,255, 0.6));
-            left: 35px;
-            position: absolute;
-            right: 130px;
-
-            .meter {
-              background: rgb(245,245,245);
-              border: 1px solid rgb(160,160,160);
-              @include box-shadow(inset 0 0 3px 0 rgba(0,0,0, 0.15));
-              @include box-sizing(border-box);
-              @include border-radius(4px);
-              height: 22px;
-              margin: 0 auto;
-              padding: 2px;
-              width: 100%;
-
-              .meter-fill {
-                background: $blue;
-                @include background-image(linear-gradient(-45deg, rgba(255,255,255, 0.15) 25%,
-                                                                  transparent 25%,
-                                                                  transparent 50%,
-                                                                  rgba(255,255,255, 0.15) 50%,
-                                                                  rgba(255,255,255, 0.15) 75%,
-                                                                  transparent 75%));
-                background-size: 40px 40px;
-                background-repeat: repeat-x;
-                border: 1px solid rgb(115,115,115);
-                @include border-radius(4px);
-                @include box-sizing(border-box);
-                content: "";
-                display: block;
-                height: 100%;
-                width: 60%;
-              }
-            }
-          }
+        .enter-course {
+          @include button(shiny, $blue);
+          @include box-sizing(border-box);
+          @include border-radius(3px);
+          display: block;
+          float: left;
+          font: normal 1rem/1.6rem $sans-serif;
+          letter-spacing: 1px;
+          padding: 6px 0px;
+          text-transform: uppercase;
+          text-align: center;
+          margin-top: 16px;
+          width: flex-grid(4);
         }
       }
 
diff --git a/lms/templates/dashboard.html b/lms/templates/dashboard.html
index b73e3a57bc..0f9c26611b 100644
--- a/lms/templates/dashboard.html
+++ b/lms/templates/dashboard.html
@@ -85,19 +85,9 @@
                 <h3>${course.number} ${course.title}</h3>
               </hgroup>
               <section class="course-status">
-                <p>Class Starts - <span>${course.start_date_text}</span></div>
-              </section>
-              <section class="meta">
-                <div class="course-work-icon"></div>
-                <div class="progress">
-                  <div class="meter">
-                    <div class="meter-fill"></div>
-                  </div>
-                </div>
-                <div class="complete">
-                  ##<p><span class="completeness">60%</span> complete</p>
-                </div>
+                <p>Class Starts - <span>${course.start_date_text}</span></p>
               </section>
+              <p class="enter-course">View Courseware</p>
             </section>
           </a>
         </article>
diff --git a/lms/templates/portal/course_about.html b/lms/templates/portal/course_about.html
index a7e7abad20..bff24d597a 100644
--- a/lms/templates/portal/course_about.html
+++ b/lms/templates/portal/course_about.html
@@ -74,7 +74,7 @@
 	        %if show_link:
                 <a href="${course_target}">
                 %endif
-                <span class="register disabled">You are registered for this course (${course.number}). <strong>View Courseware.<strong></span>
+                <span class="register disabled">You are registered for this course (${course.number})</span> <strong>View Courseware</strong>
 	        %if show_link:
                 </a>
                 %endif

From f2fc51bd35f764d379112b5cec0c6dbbb7084505 Mon Sep 17 00:00:00 2001
From: Kyle Fiedler <kfiedler@thoughtbot.com>
Date: Fri, 31 Aug 2012 16:08:41 -0400
Subject: [PATCH 128/181] Added class problem-and-answer with margin on the
 bottom

---
 common/lib/xmodule/xmodule/css/capa/display.scss | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/common/lib/xmodule/xmodule/css/capa/display.scss b/common/lib/xmodule/xmodule/css/capa/display.scss
index a368167448..212d1cc649 100644
--- a/common/lib/xmodule/xmodule/css/capa/display.scss
+++ b/common/lib/xmodule/xmodule/css/capa/display.scss
@@ -32,6 +32,7 @@ section.problem {
       display: inline;
   }
 
+
   div {
     p {
       &.answer {
@@ -321,6 +322,10 @@ section.problem {
     }
   }
 
+  .problem-and-answer {
+    margin-bottom: lh();
+  }
+
   .CodeMirror {
     border: 1px solid black;
     font-size: 14px;

From 60ea6215608aefd320503cd2ae565e5c3532b352 Mon Sep 17 00:00:00 2001
From: Kyle Fiedler <kfiedler@thoughtbot.com>
Date: Fri, 31 Aug 2012 16:55:50 -0400
Subject: [PATCH 129/181] Remove problem-answer class that will now not be used

---
 common/lib/xmodule/xmodule/css/capa/display.scss | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/common/lib/xmodule/xmodule/css/capa/display.scss b/common/lib/xmodule/xmodule/css/capa/display.scss
index 212d1cc649..aea59a4d63 100644
--- a/common/lib/xmodule/xmodule/css/capa/display.scss
+++ b/common/lib/xmodule/xmodule/css/capa/display.scss
@@ -322,10 +322,6 @@ section.problem {
     }
   }
 
-  .problem-and-answer {
-    margin-bottom: lh();
-  }
-
   .CodeMirror {
     border: 1px solid black;
     font-size: 14px;

From e8c4bc0c2513b5d6f280e130c936f695f323ce72 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Fri, 31 Aug 2012 14:37:31 -0400
Subject: [PATCH 130/181] Accordion distinguishes items to be graded

---
 lms/djangoapps/courseware/module_render.py | 4 +++-
 lms/templates/accordion.html               | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py
index ee29491d27..393419af53 100644
--- a/lms/djangoapps/courseware/module_render.py
+++ b/lms/djangoapps/courseware/module_render.py
@@ -98,7 +98,9 @@ def toc_for_course(user, request, course, active_chapter, active_section, course
                                  'url_name': section.url_name,
                                  'format': section.metadata.get('format', ''),
                                  'due': section.metadata.get('due', ''),
-                                 'active': active})
+                                 'active': active,
+                                 'graded': section.metadata.get('graded', False),
+                                 })
 
         chapters.append({'display_name': chapter.display_name,
                          'url_name': chapter.url_name,
diff --git a/lms/templates/accordion.html b/lms/templates/accordion.html
index 0d3dee9495..fa91927462 100644
--- a/lms/templates/accordion.html
+++ b/lms/templates/accordion.html
@@ -7,7 +7,7 @@
 
     <ul>
       % for section in chapter['sections']:
-          <li${' class="active"' if 'active' in section and section['active'] else ''}>
+          <li class="${'active' if 'active' in section and section['active'] else ''} ${'graded'  if 'graded' in section and section['graded'] else ''}">
             <a href="${reverse('courseware_section', args=[course_id, chapter['url_name'], section['url_name']])}">
               <p>${section['display_name']}
               <span class="subtitle">

From 648d54b027e020bb2b0fe53dbdfe3fb9f40b0345 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Fri, 31 Aug 2012 14:38:39 -0400
Subject: [PATCH 131/181] Adjust comments

---
 lms/djangoapps/courseware/module_render.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py
index 393419af53..b8416426b6 100644
--- a/lms/djangoapps/courseware/module_render.py
+++ b/lms/djangoapps/courseware/module_render.py
@@ -62,7 +62,7 @@ def toc_for_course(user, request, course, active_chapter, active_section, course
 
     where SECTIONS is a list
     [ {'display_name': name, 'url_name': url_name,
-       'format': format, 'due': due, 'active' : bool}, ...]
+       'format': format, 'due': due, 'active' : bool, 'graded': bool}, ...]
 
     active is set for the section and chapter corresponding to the passed
     parameters, which are expected to be url_names of the chapter+section.

From e7a7c71324b80cc8b4ff49d24eeb55f8778a1128 Mon Sep 17 00:00:00 2001
From: Kyle Fiedler <kfiedler@thoughtbot.com>
Date: Fri, 31 Aug 2012 17:58:08 -0400
Subject: [PATCH 132/181] Added icon to mark it as having a due date

---
 lms/static/images/graded.png                    | Bin 0 -> 571 bytes
 lms/static/sass/course/courseware/_sidebar.scss |  12 ++++++++++++
 2 files changed, 12 insertions(+)
 create mode 100644 lms/static/images/graded.png

diff --git a/lms/static/images/graded.png b/lms/static/images/graded.png
new file mode 100644
index 0000000000000000000000000000000000000000..a1220159f866684f1a5910f8e7a73048cb208858
GIT binary patch
literal 571
zcmV-B0>u4^P)<h;3K|Lk000e1NJLTq000mG000mO1^@s6AM^iV00064Nkl<Zc-noF
zOGq106o#WUh#*xF6e)s8yHZ4pQc$|-Mi&ZQX<O(*go-Z2jUcYOQL=L<sm5rnCPmAj
zU>`L2TnPm?;`@=ANhZ!rOjBERCQkpObLEPIfsZ@qKj;2)?mZKi%PTTkfC{!jDQON=
z0151ZD$-b@2UraEsPv6=PK#5VF)FRyXv1Ox>oY3KL~y|p%|CTfJT*PE;h%ZAKpNIL
zikFr1u^J(!-<XU@VV7lD?y_kmPD1ZmVh3^eqp9c%!~$T&8klMLsy1GvE<N@0szQ26
zAgW&-VfVM;ZC%oju$k`R9Z0dar1>P1JBo>J+m8O2@82%!Vf8FELI%`DE@;Q|Q*gE6
z<qu8XN*Jrq=l7Gw%IlCO+9t-6&q^)yPLOr$mLHDG0u?oV*CI&vfC9vaSo1;Gn}utW
z>~VRP?YU);>6emJ5R%XPIo)E<zcizd;&h+%*U^BT6~7|RlqR%iA2kmnHtf0wP93mu
zJ_hW51TbO8g?0xfwDda$b+gA2D##j>gG}Rb`=ueuSAL0Ujz{orp8C0w1we_ND+WH6
z8(2>Yjs|F};#EisQxS*3$LLZ;-E&6+qBI!Yz_`NYGs+3O6Z{9vYJ9>StMP-JkFw5c
z9SsUUMPm3|yYQAlAb=2<10LvYMU#<}dks+d0i>~*hjifMwf}?1L1^gu13Ul#002ov
JPDHLkV1n8n1%?0s

literal 0
HcmV?d00001

diff --git a/lms/static/sass/course/courseware/_sidebar.scss b/lms/static/sass/course/courseware/_sidebar.scss
index 52a639d9a6..27040bece5 100644
--- a/lms/static/sass/course/courseware/_sidebar.scss
+++ b/lms/static/sass/course/courseware/_sidebar.scss
@@ -180,6 +180,18 @@ section.course-index {
             font-weight: normal;
           }
         }
+
+        &.graded {
+         > a {
+           background-image: url('../images/graded.png');
+           background-repeat: no-repeat;
+           background-position: 97% center;
+          }
+
+          &.active > a {
+            @include background-image(url('../images/graded.png'), linear-gradient(top, #e6e6e6, #d6d6d6));
+          }
+        }
       }
     }
   }

From e482a32d276f9673031128dc3936f304e8cf23f2 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Fri, 31 Aug 2012 14:37:31 -0400
Subject: [PATCH 133/181] Accordion distinguishes items to be graded

---
 lms/djangoapps/courseware/module_render.py | 4 +++-
 lms/templates/accordion.html               | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py
index ee29491d27..393419af53 100644
--- a/lms/djangoapps/courseware/module_render.py
+++ b/lms/djangoapps/courseware/module_render.py
@@ -98,7 +98,9 @@ def toc_for_course(user, request, course, active_chapter, active_section, course
                                  'url_name': section.url_name,
                                  'format': section.metadata.get('format', ''),
                                  'due': section.metadata.get('due', ''),
-                                 'active': active})
+                                 'active': active,
+                                 'graded': section.metadata.get('graded', False),
+                                 })
 
         chapters.append({'display_name': chapter.display_name,
                          'url_name': chapter.url_name,
diff --git a/lms/templates/accordion.html b/lms/templates/accordion.html
index 0d3dee9495..fa91927462 100644
--- a/lms/templates/accordion.html
+++ b/lms/templates/accordion.html
@@ -7,7 +7,7 @@
 
     <ul>
       % for section in chapter['sections']:
-          <li${' class="active"' if 'active' in section and section['active'] else ''}>
+          <li class="${'active' if 'active' in section and section['active'] else ''} ${'graded'  if 'graded' in section and section['graded'] else ''}">
             <a href="${reverse('courseware_section', args=[course_id, chapter['url_name'], section['url_name']])}">
               <p>${section['display_name']}
               <span class="subtitle">

From 8c0780bcdc4ee9e783b67e33fbbcd04eebcdad3a Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Fri, 31 Aug 2012 14:38:39 -0400
Subject: [PATCH 134/181] Adjust comments

---
 lms/djangoapps/courseware/module_render.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py
index 393419af53..b8416426b6 100644
--- a/lms/djangoapps/courseware/module_render.py
+++ b/lms/djangoapps/courseware/module_render.py
@@ -62,7 +62,7 @@ def toc_for_course(user, request, course, active_chapter, active_section, course
 
     where SECTIONS is a list
     [ {'display_name': name, 'url_name': url_name,
-       'format': format, 'due': due, 'active' : bool}, ...]
+       'format': format, 'due': due, 'active' : bool, 'graded': bool}, ...]
 
     active is set for the section and chapter corresponding to the passed
     parameters, which are expected to be url_names of the chapter+section.

From 3a9542ebb178e8bb3ab55f3d4a6ec6746d176da7 Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Fri, 31 Aug 2012 15:11:29 -0700
Subject: [PATCH 135/181] Move function; stop shadowing map; add todo about
 caching for _DISCUSSIONINFO

---
 lms/djangoapps/django_comment_client/utils.py | 21 ++++++++++---------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/lms/djangoapps/django_comment_client/utils.py b/lms/djangoapps/django_comment_client/utils.py
index e5b32d1d9c..fd9e53b3b8 100644
--- a/lms/djangoapps/django_comment_client/utils.py
+++ b/lms/djangoapps/django_comment_client/utils.py
@@ -19,11 +19,11 @@ import urllib
 import pystache_custom as pystache
 
 
+# TODO these should be cached via django's caching rather than in-memory globals
 _FULLMODULES = None
 _DISCUSSIONINFO = None
 
 
-
 def extract(dic, keys):
     return {k: dic.get(k) for k in keys}
 
@@ -67,6 +67,16 @@ def get_discussion_category_map(course):
         initialize_discussion_info(course)
     return _DISCUSSIONINFO['category_map']
 
+def sort_map_entries(category_map):
+    things = []
+    for title, entry in category_map["entries"].items():
+        things.append((title, entry))
+    for title, category in category_map["subcategories"].items():
+        things.append((title, category))
+        sort_map_entries(category_map["subcategories"][title])
+    category_map["children"] = [x[0] for x in sorted(things, key=lambda x: x[1]["sort_key"])]
+
+
 def initialize_discussion_info(course):
 
     global _DISCUSSIONINFO
@@ -111,15 +121,6 @@ def initialize_discussion_info(course):
             node[level]["entries"][entry["title"]] = {"id": entry["id"], 
                                                       "sort_key": entry["sort_key"]}
 
-    def sort_map_entries(map):
-        things = []
-        for title, entry in map["entries"].items():
-            things.append((title, entry))
-        for title, category in map["subcategories"].items():
-            things.append((title, category))
-            sort_map_entries(map["subcategories"][title])
-        map["children"] = [x[0] for x in sorted(things, key=lambda x: x[1]["sort_key"])]
-
     sort_map_entries(category_map)
 
     _DISCUSSIONINFO = {}

From e600004b1fa6280c036b67c1a932d7e6625d5595 Mon Sep 17 00:00:00 2001
From: Ibrahim Awwal <ibrahim.awwal@gmail.com>
Date: Fri, 31 Aug 2012 15:19:17 -0700
Subject: [PATCH 136/181] Make the comment client not explode if it doesn't get
 a sort preference from the comment service.

---
 lms/djangoapps/django_comment_client/forum/views.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lms/djangoapps/django_comment_client/forum/views.py b/lms/djangoapps/django_comment_client/forum/views.py
index ae8e999a8a..2b3d140c7e 100644
--- a/lms/djangoapps/django_comment_client/forum/views.py
+++ b/lms/djangoapps/django_comment_client/forum/views.py
@@ -114,7 +114,8 @@ def get_threads(request, course_id, discussion_id=None):
         # If the user did not select a sort key, use their last used sort key
         user = cc.User.from_django_user(request.user)
         user.retrieve()
-        default_query_params['sort_key'] = user.default_sort_key
+        # TODO: After the comment service is updated this can just be user.default_sort_key because the service returns the default value
+        default_query_params['sort_key'] = user.get('default_sort_key') or default_query_params['sort_key']
     else:
         # If the user clicked a sort key, update their default sort key
         user = cc.User.from_django_user(request.user)

From 9eecd43a4a765e0af1fee81f42bd19f173382c10 Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Fri, 31 Aug 2012 17:49:15 -0700
Subject: [PATCH 137/181] Fix jump to courseware test to expect a redirect

---
 lms/djangoapps/courseware/tests/tests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lms/djangoapps/courseware/tests/tests.py b/lms/djangoapps/courseware/tests/tests.py
index 56a2384bc5..d7fffb7499 100644
--- a/lms/djangoapps/courseware/tests/tests.py
+++ b/lms/djangoapps/courseware/tests/tests.py
@@ -218,7 +218,7 @@ class PageLoader(ActivateLoginTestCase):
                                            'location': descriptor.location.url()}))
             msg = str(resp.status_code)
 
-            if resp.status_code != 200:
+            if resp.status_code != 302:
                 msg = "ERROR " + msg
                 all_ok = False
                 num_bad += 1

From 877ae91b7f7dcbcae111ad04ebbafe0826c78fe5 Mon Sep 17 00:00:00 2001
From: David Ormsbee <dave@edx.org>
Date: Fri, 31 Aug 2012 21:50:33 -0400
Subject: [PATCH 138/181] Don't show View Courseware buttons if they don't have
 access to load the courseware

---
 common/djangoapps/student/views.py     | 6 +++++-
 lms/djangoapps/courseware/views.py     | 6 +++++-
 lms/templates/dashboard.html           | 4 +++-
 lms/templates/portal/course_about.html | 5 ++++-
 4 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py
index ce15a2a003..27adf485f0 100644
--- a/common/djangoapps/student/views.py
+++ b/common/djangoapps/student/views.py
@@ -131,10 +131,14 @@ def dashboard(request):
         staff_access = True
         errored_courses = modulestore().get_errored_courses()
 
+    show_courseware_links_for = frozenset(course.id for course in courses
+                                          if has_access(request.user, course, 'load'))
+
     context = {'courses': courses,
                'message': message,
                'staff_access': staff_access,
-               'errored_courses': errored_courses,}
+               'errored_courses': errored_courses,
+               'show_courseware_links_for' : show_courseware_links_for}
 
     return render_to_response('dashboard.html', context)
 
diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py
index 6122ebd333..71ec687cf6 100644
--- a/lms/djangoapps/courseware/views.py
+++ b/lms/djangoapps/courseware/views.py
@@ -264,7 +264,11 @@ def registered_for_course(course, user):
 def course_about(request, course_id):
     course = get_course_with_access(request.user, course_id, 'see_exists')
     registered = registered_for_course(course, request.user)
-    return render_to_response('portal/course_about.html', {'course': course, 'registered': registered})
+    show_courseware_link = has_access(request.user, course, 'load')
+    return render_to_response('portal/course_about.html',
+                              {'course': course,
+                               'registered': registered, 
+                               'show_courseware_link' : show_courseware_link})
 
 
 @ensure_csrf_cookie
diff --git a/lms/templates/dashboard.html b/lms/templates/dashboard.html
index 0f9c26611b..6be72277cc 100644
--- a/lms/templates/dashboard.html
+++ b/lms/templates/dashboard.html
@@ -87,7 +87,9 @@
               <section class="course-status">
                 <p>Class Starts - <span>${course.start_date_text}</span></p>
               </section>
-              <p class="enter-course">View Courseware</p>
+              % if course.id in show_courseware_links_for:
+                <p class="enter-course">View Courseware</p>
+              % endif
             </section>
           </a>
         </article>
diff --git a/lms/templates/portal/course_about.html b/lms/templates/portal/course_about.html
index bff24d597a..d6c731431e 100644
--- a/lms/templates/portal/course_about.html
+++ b/lms/templates/portal/course_about.html
@@ -74,7 +74,10 @@
 	        %if show_link:
                 <a href="${course_target}">
                 %endif
-                <span class="register disabled">You are registered for this course (${course.number})</span> <strong>View Courseware</strong>
+                <span class="register disabled">You are registered for this course (${course.number})</span>
+                % if show_courseware_link:
+                  <strong>View Courseware</strong>
+                % endif
 	        %if show_link:
                 </a>
                 %endif

From 6f03ebb198af88cb1cf57f16a880897646dada3e Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Sat, 1 Sep 2012 08:26:54 -0400
Subject: [PATCH 139/181] Add message for student in CustomResponse error

---
 common/lib/capa/capa/responsetypes.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index d1b58e53a9..13a0bec3a2 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -726,7 +726,7 @@ class NumericalResponse(LoncapaResponse):
         # I think this is just pyparsing.ParseException, calc.UndefinedVariable:
         # But we'd need to confirm
         except:
-            raise StudentInputError("Invalid input: could not parse '%s' as a number" %\
+            raise StudentInputError("Invalid input: could not interpret '%s' as a number" %\
                                     cgi.escape(student_answer))
 
         if correct:
@@ -920,6 +920,7 @@ def sympy_check2():
                 print "oops in customresponse (code) error %s" % err
                 print "context = ", self.context
                 print traceback.format_exc()
+                raise StudentInputError("Error: Problem could not be evaluated with your input") # Notify student
         else:					# self.code is not a string; assume its a function
 
             # this is an interface to the Tutor2 check functions

From 90bcb00e5885b95d879a70e7c84ef81b2153a523 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Sat, 1 Sep 2012 08:56:16 -0400
Subject: [PATCH 140/181] Allow ckt editor to be embedded in sequence

---
 common/lib/xmodule/xmodule/js/src/sequence/display.coffee | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/common/lib/xmodule/xmodule/js/src/sequence/display.coffee b/common/lib/xmodule/xmodule/js/src/sequence/display.coffee
index ab337d9b7e..1c3347c8c0 100644
--- a/common/lib/xmodule/xmodule/js/src/sequence/display.coffee
+++ b/common/lib/xmodule/xmodule/js/src/sequence/display.coffee
@@ -87,6 +87,8 @@ class @Sequence
       XModule.loadModules('display', @$('#seq_content'))
 
       MathJax.Hub.Queue(["Typeset", MathJax.Hub, "seq_content"]) # NOTE: Actually redundant. Some other MathJax call also being performed
+      window.update_schematics()
+
       @position = new_position
       @toggleArrows()
       @hookUpProgressEvent()

From ea4019c0a043551589d849a0f10c20dcfb6e6cf8 Mon Sep 17 00:00:00 2001
From: kimth <kimt@mit.edu>
Date: Sat, 1 Sep 2012 09:01:19 -0400
Subject: [PATCH 141/181] Add comment to sequence/display.coffee

---
 common/lib/xmodule/xmodule/js/src/sequence/display.coffee | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/lib/xmodule/xmodule/js/src/sequence/display.coffee b/common/lib/xmodule/xmodule/js/src/sequence/display.coffee
index 1c3347c8c0..25d69ccca3 100644
--- a/common/lib/xmodule/xmodule/js/src/sequence/display.coffee
+++ b/common/lib/xmodule/xmodule/js/src/sequence/display.coffee
@@ -87,7 +87,7 @@ class @Sequence
       XModule.loadModules('display', @$('#seq_content'))
 
       MathJax.Hub.Queue(["Typeset", MathJax.Hub, "seq_content"]) # NOTE: Actually redundant. Some other MathJax call also being performed
-      window.update_schematics()
+      window.update_schematics() # For embedded circuit simulator exercises in 6.002x
 
       @position = new_position
       @toggleArrows()

From 9433e897106c2916066340ca1cfdfc026d120d46 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Sat, 1 Sep 2012 10:16:38 -0400
Subject: [PATCH 142/181] add docs for gitreload

---
 doc/development.md | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/doc/development.md b/doc/development.md
index cb71278c40..fcf0b7a40f 100644
--- a/doc/development.md
+++ b/doc/development.md
@@ -72,3 +72,25 @@ Very handy: if you uncomment the `--pdb` argument in `NOSE_ARGS` in `lms/envs/te
 If you change course content, while running the LMS in dev mode, it is unnecessary to restart to refresh the modulestore.  
 
 Instead, hit /migrate/modules to see a list of all modules loaded, and click on links (eg /migrate/reload/edx4edx) to reload a course.
+
+### Gitreload-based workflow
+
+github (or other equivalent git-based repository systems) used for
+course content can be setup to trigger an automatic reload when changes are pushed.  Here is how:
+
+1. Each content directory in mitx_all/data should be a clone of a git repo
+
+2. The user running the mitx gunicorn process should have its ssh key registered with the git repo
+
+3. The list settings.ALLOWED_GITRELOAD_IPS should contain the IP address of the git repo originating the gitreload request.
+    By default, this list is ['207.97.227.253', '50.57.128.197', '108.171.174.178'] (the github IPs).
+    The list can be overridden in the startup file used, eg lms/envs/dev*.py
+
+4. The git post-receive-hook should POST to /gitreload with a JSON payload.  This payload should define at least
+
+   { "repository" : { "name" : reload_dir }
+
+    where reload_dir is the directory name of the content to reload (ie mitx_all/data/reload_dir should exist)
+
+    The mitx server will then do "git reset --hard HEAD; git clean -f -d; git pull origin" in that directory.  After the pull,
+    it will reload the modulestore for that course.

From b64696dbd0c6eaa3ec50a4d98954100d125c17d5 Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Sun, 2 Sep 2012 02:05:50 -0700
Subject: [PATCH 143/181] Allowing customresponse messages to be passed to the
 inputtype

---
 common/lib/capa/capa/responsetypes.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index d1b58e53a9..ee75613ea1 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -916,6 +916,7 @@ def sympy_check2():
             try:
                 exec self.code in self.context['global_context'], self.context
                 correct = self.context['correct']
+                messages = self.context['messages']
             except Exception as err:
                 print "oops in customresponse (code) error %s" % err
                 print "context = ", self.context

From 08693a1b4195c5140f5868c326d179840f5de72f Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Sun, 2 Sep 2012 02:52:59 -0700
Subject: [PATCH 144/181] Make customresponse respect point values on input
 fields

---
 common/lib/capa/capa/responsetypes.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py
index ee75613ea1..549ab73850 100644
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -970,7 +970,8 @@ def sympy_check2():
         # build map giving "correct"ness of the answer(s)
         correct_map = CorrectMap()
         for k in range(len(idset)):
-            correct_map.set(idset[k], correct[k], msg=messages[k])
+            correct_map.set(idset[k], correct[k], msg=messages[k],
+                            npoints=self.maxpoints[idset[k]])
         return correct_map
 
     def get_answers(self):

From 1a1c5edab465b49c4042d83e463bb3ddb24c2db3 Mon Sep 17 00:00:00 2001
From: David Ormsbee <dave@edx.org>
Date: Sun, 2 Sep 2012 16:56:17 -0400
Subject: [PATCH 145/181] Completely cosmetic fix to make the indentation
 clearer.

---
 .../courseware/course_navigation.html         | 42 +++++++++----------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/lms/templates/courseware/course_navigation.html b/lms/templates/courseware/course_navigation.html
index 5df09207b1..92c9632625 100644
--- a/lms/templates/courseware/course_navigation.html
+++ b/lms/templates/courseware/course_navigation.html
@@ -19,34 +19,34 @@ def url_class(url):
     <ol class="course-tabs">
       <li class="courseware"><a href="${reverse('courseware', args=[course.id])}" class="${url_class('courseware')}">Courseware</a></li>
       <li class="info"><a href="${reverse('info', args=[course.id])}" class="${url_class('info')}">Course Info</a></li>
-% if hasattr(course,'syllabus_present') and course.syllabus_present:
+    % if hasattr(course,'syllabus_present') and course.syllabus_present:
       <li class="syllabus"><a href="${reverse('syllabus', args=[course.id])}" class="${url_class('syllabus')}">Syllabus</a></li>
-% endif
-% if user.is_authenticated():
-  % if settings.MITX_FEATURES.get('ENABLE_TEXTBOOK'):
+    % endif
+    % if user.is_authenticated():
+      % if settings.MITX_FEATURES.get('ENABLE_TEXTBOOK'):
         % for index, textbook in enumerate(course.textbooks):
             <li class="book"><a href="${reverse('book', args=[course.id, index])}" class="${url_class('book')}">${textbook.title}</a></li>
         % endfor
-  % endif
-  % if settings.MITX_FEATURES.get('ENABLE_DISCUSSION_SERVICE'):
+      % endif
+      % if settings.MITX_FEATURES.get('ENABLE_DISCUSSION_SERVICE'):
         <li class="discussion"><a href="${reverse('django_comment_client.forum.views.forum_form_discussion', args=[course.id])}" class="${url_class('discussion')}">Discussion</a></li>
         ## <li class="news"><a href="${reverse('news', args=[course.id])}" class="${url_class('news')}">News</a></li>
-  % endif 
-        
-  % if settings.MITX_FEATURES.get('ENABLE_DISCUSSION'):
-        <li class="discussion"><a href="${reverse('questions')}">Discussion</a></li>
-  % endif
-% endif
-% if settings.WIKI_ENABLED:
-      <li class="wiki"><a href="${reverse('course_wiki', args=[course.id])}" class="${url_class('wiki')}">Wiki</a></li>
-% endif
-% if user.is_authenticated():
-      <li class="profile"><a href="${reverse('progress', args=[course.id])}" class="${url_class('progress')}">Progress</a></li>
-% endif
-% if staff_access:
-      <li class="instructor"><a href="${reverse('instructor_dashboard', args=[course.id])}" class="${url_class('instructor')}">Instructor</a></li>
-% endif
+      % endif 
 
+      % if settings.MITX_FEATURES.get('ENABLE_DISCUSSION'):
+        <li class="discussion"><a href="${reverse('questions')}">Discussion</a></li>
+      % endif
+    % endif
+
+    % if settings.WIKI_ENABLED:
+      <li class="wiki"><a href="${reverse('course_wiki', args=[course.id])}" class="${url_class('wiki')}">Wiki</a></li>
+    % endif
+    % if user.is_authenticated():
+      <li class="profile"><a href="${reverse('progress', args=[course.id])}" class="${url_class('progress')}">Progress</a></li>
+    % endif
+    % if staff_access:
+      <li class="instructor"><a href="${reverse('instructor_dashboard', args=[course.id])}" class="${url_class('instructor')}">Instructor</a></li>
+    % endif
     </ol>
   </div>
 </nav>

From 7664f910b1428084d4e0eff52cb25103f6eaa565 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Sun, 2 Sep 2012 19:04:55 -0400
Subject: [PATCH 146/181] add is_released

---
 common/djangoapps/xmodule_modifiers.py | 12 +++++++++++-
 lms/templates/staff_problem_info.html  |  1 +
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/common/djangoapps/xmodule_modifiers.py b/common/djangoapps/xmodule_modifiers.py
index 17380bff18..10cce1a95b 100644
--- a/common/djangoapps/xmodule_modifiers.py
+++ b/common/djangoapps/xmodule_modifiers.py
@@ -1,6 +1,7 @@
 import re
 import json
 import logging
+import time
 
 from django.conf import settings
 from functools import wraps
@@ -117,6 +118,13 @@ def add_histogram(get_html, module, user):
             data_dir = ""
         source_file = module.metadata.get('source_file','')	# source used to generate the problem XML, eg latex or word
 
+        # useful to indicate to staff if problem has been released or not
+        now = time.gmtime()
+        is_released = "unknown"
+        if hasattr(module,'start'):
+            if module.start is not None:
+                is_released = "<font color='red'>Yes!</font>" if (now > module.start) else "<font color='green'>Not yet</font>"
+        
         staff_context = {'definition': module.definition.get('data'),
                          'metadata': json.dumps(module.metadata, indent=4),
                          'location': module.location,
@@ -130,7 +138,9 @@ def add_histogram(get_html, module, user):
                          'xqa_server' : settings.MITX_FEATURES.get('USE_XQA_SERVER','http://xqa:server@content-qa.mitx.mit.edu/xqa'),
                          'histogram': json.dumps(histogram),
                          'render_histogram': render_histogram,
-                         'module_content': get_html()}
+                         'module_content': get_html(),
+                         'is_released': is_released,
+                         }
         return render_to_string("staff_problem_info.html", staff_context)
 
     return _get_html
diff --git a/lms/templates/staff_problem_info.html b/lms/templates/staff_problem_info.html
index 47194aa6fd..6d9d1a3a30 100644
--- a/lms/templates/staff_problem_info.html
+++ b/lms/templates/staff_problem_info.html
@@ -32,6 +32,7 @@ ${module_content}
       <h2>Staff Debug</h2>
     </header>
     <div class="staff_info" style="display:block">
+is_released = ${is_released}
 location = ${location | h}
 github = <a href="${edit_link}">${edit_link | h}</a>
 %if source_file:

From 2efe481237d0f60c36be9fbba756447d83a3965d Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Sun, 2 Sep 2012 19:05:34 -0400
Subject: [PATCH 147/181] move instructor dashboard into its own lms djangoapp;
 add new functionality - grade dump and download as csv, manage staff list,
 force reload of course from xml

---
 common/djangoapps/student/models.py           |   3 +
 lms/djangoapps/courseware/access.py           |  52 ++-
 lms/djangoapps/courseware/grades.py           |  13 +-
 lms/djangoapps/courseware/views.py            |  93 -----
 lms/djangoapps/instructor/__init__.py         |   0
 lms/djangoapps/instructor/views.py            | 330 ++++++++++++++++++
 lms/envs/common.py                            |   1 +
 .../courseware/instructor_dashboard.html      |  81 +++++
 lms/urls.py                                   |   8 +-
 9 files changed, 474 insertions(+), 107 deletions(-)
 create mode 100644 lms/djangoapps/instructor/__init__.py
 create mode 100644 lms/djangoapps/instructor/views.py

diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py
index 9cde878d21..c494035104 100644
--- a/common/djangoapps/student/models.py
+++ b/common/djangoapps/student/models.py
@@ -184,6 +184,9 @@ class CourseEnrollment(models.Model):
     class Meta:
         unique_together = (('user', 'course_id'), )
 
+    def __unicode__(self):
+        return "%s: %s (%s)" % (self.user,self.course_id,self.created)
+
 @receiver(post_save, sender=CourseEnrollment)
 def assign_default_role(sender, instance, **kwargs):
     if instance.user.is_staff:
diff --git a/lms/djangoapps/courseware/access.py b/lms/djangoapps/courseware/access.py
index 91c769f90a..4f35aa98fa 100644
--- a/lms/djangoapps/courseware/access.py
+++ b/lms/djangoapps/courseware/access.py
@@ -30,7 +30,7 @@ def has_access(user, obj, action):
     Things this module understands:
     - start dates for modules
     - DISABLE_START_DATES
-    - different access for staff, course staff, and students.
+    - different access for instructor, staff, course staff, and students.
 
     user: a Django user object. May be anonymous.
 
@@ -70,6 +70,20 @@ def has_access(user, obj, action):
     raise TypeError("Unknown object type in has_access(): '{0}'"
                     .format(type(obj)))
 
+def get_access_group_name(obj,action):
+    '''
+    Returns group name for user group which has "action" access to the given object.
+
+    Used in managing access lists.
+    '''
+
+    if isinstance(obj, CourseDescriptor):
+        return _get_access_group_name_course_desc(obj, action)
+
+    # Passing an unknown object here is a coding error, so rather than
+    # returning a default, complain.
+    raise TypeError("Unknown object type in get_access_group_name(): '{0}'"
+                    .format(type(obj)))
 
 # ================ Implementation helpers ================================
 
@@ -138,11 +152,19 @@ def _has_access_course_desc(user, course, action):
         'load': can_load,
         'enroll': can_enroll,
         'see_exists': see_exists,
-        'staff': lambda: _has_staff_access_to_descriptor(user, course)
+        'staff': lambda: _has_staff_access_to_descriptor(user, course),
+        'instructor': lambda: _has_staff_access_to_descriptor(user, course, require_instructor=True),
         }
 
     return _dispatch(checkers, action, user, course)
 
+def _get_access_group_name_course_desc(course, action):
+    '''
+    Return name of group which gives staff access to course.  Only understands action = 'staff'
+    '''
+    if not action=='staff':
+        return []
+    return _course_staff_group_name(course.location)
 
 def _has_access_error_desc(user, descriptor, action):
     """
@@ -292,6 +314,15 @@ def _course_staff_group_name(location):
     """
     return 'staff_%s' % Location(location).course
 
+def _course_instructor_group_name(location):
+    """
+    Get the name of the instructor group for a location.  Right now, that's instructor_COURSE.
+    A course instructor has all staff privileges, but also can manage list of course staff (add, remove, list).
+
+    location: something that can passed to Location.
+    """
+    return 'instructor_%s' % Location(location).course
+
 def _has_global_staff_access(user):
     if user.is_staff:
         debug("Allow: user.is_staff")
@@ -301,11 +332,13 @@ def _has_global_staff_access(user):
         return False
 
 
-def _has_staff_access_to_location(user, location):
+def _has_staff_access_to_location(user, location, require_instructor=False):
     '''
     Returns True if the given user has staff access to a location.  For now this
     is equivalent to having staff access to the course location.course.
 
+    If require_instructor=True, then user must be in instructor_* group.
+
     This means that user is in the staff_* group, or is an overall admin.
 
     TODO (vshnayder): this needs to be changed to allow per-course_id permissions, not per-course
@@ -323,8 +356,13 @@ def _has_staff_access_to_location(user, location):
     # If not global staff, is the user in the Auth group for this class?
     user_groups = [g.name for g in user.groups.all()]
     staff_group = _course_staff_group_name(location)
-    if staff_group in user_groups:
-        debug("Allow: user in group %s", staff_group)
+    if not require_instructor:
+        if staff_group in user_groups:
+            debug("Allow: user in group %s", staff_group)
+            return True
+    instructor_group = _course_instructor_group_name(location)
+    if instructor_group in user_groups:
+        debug("Allow: user in group %s", instructor_group)
         return True
     debug("Deny: user not in group %s", staff_group)
     return False
@@ -335,11 +373,11 @@ def _has_staff_access_to_course_id(user, course_id):
     return _has_staff_access_to_location(user, loc)
 
 
-def _has_staff_access_to_descriptor(user, descriptor):
+def _has_staff_access_to_descriptor(user, descriptor, require_instructor=False):
     """Helper method that checks whether the user has staff access to
     the course of the location.
 
     location: something that can be passed to Location
     """
-    return _has_staff_access_to_location(user, descriptor.location)
+    return _has_staff_access_to_location(user, descriptor.location, require_instructor=require_instructor)
 
diff --git a/lms/djangoapps/courseware/grades.py b/lms/djangoapps/courseware/grades.py
index 7f28f3ca5c..f32da532df 100644
--- a/lms/djangoapps/courseware/grades.py
+++ b/lms/djangoapps/courseware/grades.py
@@ -24,7 +24,7 @@ def yield_module_descendents(module):
         stack.extend( next_module.get_display_items() )
         yield next_module
 
-def grade(student, request, course, student_module_cache=None):
+def grade(student, request, course, student_module_cache=None, keep_raw_scores=False):
     """
     This grades a student as quickly as possible. It retuns the
     output from the course grader, augmented with the final letter
@@ -38,11 +38,13 @@ def grade(student, request, course, student_module_cache=None):
         up the grade. (For display)
     - grade_breakdown : A breakdown of the major components that
         make up the final grade. (For display)
+    - keep_raw_scores : if True, then value for key 'raw_scores' contains scores for every graded module
 
     More information on the format is in the docstring for CourseGrader.
     """
 
     grading_context = course.grading_context
+    raw_scores = []
 
     if student_module_cache == None:
         student_module_cache = StudentModuleCache(course.id, student, grading_context['all_descriptors'])
@@ -83,7 +85,7 @@ def grade(student, request, course, student_module_cache=None):
                     if correct is None and total is None:
                         continue
 
-                    if settings.GENERATE_PROFILE_SCORES:
+                    if settings.GENERATE_PROFILE_SCORES:	# for debugging!
                         if total > 1:
                             correct = random.randrange(max(total - 2, 1), total + 1)
                         else:
@@ -97,6 +99,8 @@ def grade(student, request, course, student_module_cache=None):
                     scores.append(Score(correct, total, graded, module.metadata.get('display_name')))
 
                 section_total, graded_total = graders.aggregate_scores(scores, section_name)
+                if keep_raw_scores:
+                    raw_scores += scores
             else:
                 section_total = Score(0.0, 1.0, False, section_name)
                 graded_total = Score(0.0, 1.0, True, section_name)
@@ -117,7 +121,10 @@ def grade(student, request, course, student_module_cache=None):
 
     letter_grade = grade_for_percentage(course.grade_cutoffs, grade_summary['percent'])
     grade_summary['grade'] = letter_grade
-
+    grade_summary['totaled_scores'] = totaled_scores	# make this available, eg for instructor download & debugging
+    if keep_raw_scores:
+        grade_summary['raw_scores'] = raw_scores        # way to get all RAW scores out to instructor
+                                                        # so grader can be double-checked
     return grade_summary
 
 def grade_for_percentage(grade_cutoffs, percentage):
diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py
index 71ec687cf6..aa3444b193 100644
--- a/lms/djangoapps/courseware/views.py
+++ b/lms/djangoapps/courseware/views.py
@@ -361,96 +361,3 @@ def progress(request, course_id, student_id=None):
 
 
 
-# ======== Instructor views =============================================================================
-
-@cache_control(no_cache=True, no_store=True, must_revalidate=True)
-def gradebook(request, course_id):
-    """
-    Show the gradebook for this course:
-    - only displayed to course staff
-    - shows students who are enrolled.
-    """
-    course = get_course_with_access(request.user, course_id, 'staff')
-
-    enrolled_students = User.objects.filter(courseenrollment__course_id=course_id).order_by('username')
-
-    # TODO (vshnayder): implement pagination.
-    enrolled_students = enrolled_students[:1000]   # HACK!
-
-    student_info = [{'username': student.username,
-                     'id': student.id,
-                     'email': student.email,
-                     'grade_summary': grades.grade(student, request, course),
-                     'realname': UserProfile.objects.get(user=student).name
-                     }
-                     for student in enrolled_students]
-
-    return render_to_response('courseware/gradebook.html', {'students': student_info,
-                                                 'course': course,
-                                                 'course_id': course_id,
-                                                 # Checked above
-                                                 'staff_access': True,})
-
-
-@cache_control(no_cache=True, no_store=True, must_revalidate=True)
-def grade_summary(request, course_id):
-    """Display the grade summary for a course."""
-    course = get_course_with_access(request.user, course_id, 'staff')
-
-    # For now, just a static page
-    context = {'course': course,
-               'staff_access': True,}
-    return render_to_response('courseware/grade_summary.html', context)
-
-
-@cache_control(no_cache=True, no_store=True, must_revalidate=True)
-def instructor_dashboard(request, course_id):
-    """Display the instructor dashboard for a course."""
-    course = get_course_with_access(request.user, course_id, 'staff')
-
-    # For now, just a static page
-    context = {'course': course,
-               'staff_access': True,}
-
-    return render_to_response('courseware/instructor_dashboard.html', context)
-
-@ensure_csrf_cookie
-@cache_control(no_cache=True, no_store=True, must_revalidate=True)
-def enroll_students(request, course_id):
-    ''' Allows a staff member to enroll students in a course.
-
-    This is a short-term hack for Berkeley courses launching fall
-    2012. In the long term, we would like functionality like this, but
-    we would like both the instructor and the student to agree. Right
-    now, this allows any instructor to add students to their course,
-    which we do not want.
-
-    It is poorly written and poorly tested, but it's designed to be
-    stripped out.
-    '''
-
-    course = get_course_with_access(request.user, course_id, 'staff')
-    existing_students = [ce.user.email for ce in CourseEnrollment.objects.filter(course_id = course_id)]
-
-    if 'new_students' in request.POST:
-        new_students = request.POST['new_students'].split('\n')
-    else:
-        new_students = []
-    new_students = [s.strip() for s in new_students]
-
-    added_students = []
-    rejected_students = []
-
-    for student in new_students:
-        try:
-            nce = CourseEnrollment(user=User.objects.get(email = student), course_id = course_id)
-            nce.save()
-            added_students.append(student)
-        except:
-            rejected_students.append(student)
-
-    return render_to_response("enroll_students.html", {'course':course_id,
-                                                       'existing_students': existing_students,
-                                                       'added_students': added_students,
-                                                       'rejected_students': rejected_students,
-                                                       'debug':new_students})
diff --git a/lms/djangoapps/instructor/__init__.py b/lms/djangoapps/instructor/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/lms/djangoapps/instructor/views.py b/lms/djangoapps/instructor/views.py
new file mode 100644
index 0000000000..ccf40cd5e5
--- /dev/null
+++ b/lms/djangoapps/instructor/views.py
@@ -0,0 +1,330 @@
+# ======== Instructor views =============================================================================
+
+import csv
+import json
+import logging
+import urllib
+import itertools
+
+from functools import partial
+from collections import defaultdict
+
+from django.conf import settings
+from django.core.context_processors import csrf
+from django.core.urlresolvers import reverse
+from django.contrib.auth.models import User, Group
+from django.contrib.auth.decorators import login_required
+from django.http import Http404, HttpResponse
+from django.shortcuts import redirect
+from mitxmako.shortcuts import render_to_response, render_to_string
+#from django.views.decorators.csrf import ensure_csrf_cookie
+from django_future.csrf import ensure_csrf_cookie
+from django.views.decorators.cache import cache_control
+
+from courseware import grades
+from courseware.access import has_access, get_access_group_name
+from courseware.courses import (get_course_with_access, get_courses_by_university)
+from student.models import UserProfile
+
+from student.models import UserTestGroup, CourseEnrollment
+from util.cache import cache, cache_if_anonymous
+from xmodule.course_module import CourseDescriptor
+from xmodule.modulestore import Location
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, NoPathToItem
+from xmodule.modulestore.search import path_to_location
+
+log = logging.getLogger("mitx.courseware")
+
+template_imports = {'urllib': urllib}
+
+@ensure_csrf_cookie
+@cache_control(no_cache=True, no_store=True, must_revalidate=True)
+def instructor_dashboard(request, course_id):
+    """Display the instructor dashboard for a course."""
+    course = get_course_with_access(request.user, course_id, 'staff')
+
+    instructor_access = has_access(request.user, course, 'instructor')	# an instructor can manage staff lists
+
+    msg = ''
+    # msg += ('POST=%s' % dict(request.POST)).replace('<','&lt;')
+
+    def escape(s):
+        """escape HTML special characters in string"""
+        return str(s).replace('<','&lt;').replace('>','&gt;')
+
+    # assemble some course statistics for output to instructor
+    datatable = {'header': ['Statistic','Value'],
+                 'title': 'Course Statistics At A Glance',
+                 }
+    data = [ ['# Enrolled' ,CourseEnrollment.objects.filter(course_id=course_id).count()] ]
+    data += compute_course_stats(course).items()
+    if request.user.is_staff:
+        data.append(['metadata', escape(str(course.metadata))])
+    datatable['data'] = data
+
+    def return_csv(fn,datatable):
+        response = HttpResponse(mimetype='text/csv')
+        response['Content-Disposition'] = 'attachment; filename=%s' % fn
+        writer = csv.writer(response,dialect='excel',quotechar='"', quoting=csv.QUOTE_ALL)
+        writer.writerow(datatable['header'])
+        for datarow in datatable['data']:
+            writer.writerow(datarow)
+        return response
+
+    def get_staff_group(course):
+        staffgrp = get_access_group_name(course,'staff')
+        try:
+            group = Group.objects.get(name=staffgrp)
+        except Group.DoesNotExist:
+            group = Group(name=staffgrp)	# create the group
+            group.save()
+        return group
+
+    # process actions from form POST
+    action = request.POST.get('action','')
+
+    if 'Reload' in action:
+        log.debug('reloading %s (%s)' % (course_id,course))
+        try:
+            data_dir = course.metadata['data_dir']
+            modulestore().try_load_course(data_dir)
+            msg += "<br/><p>Course reloaded from %s</p>" % data_dir
+        except Exception as err:
+            msg += '<br/><p>Error: %s</p>' % escape(err)
+
+    elif action=='Dump list of enrolled students':
+        log.debug(action)
+        datatable = get_student_grade_summary_data(request, course, course_id, get_grades=False)
+        datatable['title'] = 'List of students enrolled in %s' % course_id
+
+    elif 'Dump Grades' in action:
+        log.debug(action)
+        datatable = get_student_grade_summary_data(request, course, course_id, get_grades=True)
+        datatable['title'] = 'Summary Grades of students enrolled in %s' % course_id
+
+    elif 'Dump all RAW grades' in action:
+        log.debug(action)
+        datatable = get_student_grade_summary_data(request, course, course_id, get_grades=True,
+                                                   get_raw_scores=True)
+        datatable['title'] = 'Raw Grades of students enrolled in %s' % course_id
+
+    elif 'Download CSV of all student grades' in action:
+        return return_csv('grades_%s.csv' % course_id,
+                          get_student_grade_summary_data(request, course, course_id))
+
+    elif 'Download CSV of all RAW grades' in action:
+        return return_csv('grades_%s_raw.csv' % course_id,
+                          get_student_grade_summary_data(request, course, course_id, get_raw_scores=True))
+
+    elif 'List course staff' in action:
+        group = get_staff_group(course)
+        msg += 'Staff group = %s' % group.name
+        log.debug('staffgrp=%s' % group.name)
+        uset = group.user_set.all()
+        datatable = {'header': ['Username','Full name']}
+        datatable['data'] = [[ x.username, x.profile.name ] for x in uset]
+        datatable['title'] = 'List of Staff in course %s' % course_id
+
+    elif action=='Add course staff':
+        uname = request.POST['staffuser']
+        try:
+            user = User.objects.get(username=uname)
+        except User.DoesNotExist:
+            msg += '<font color="red">Error: unknown username "%s"</font>' % uname
+            user = None
+        if user is not None:
+            group = get_staff_group(course)
+            msg += '<font color="green">Added %s to staff group = %s</font>' % (user,group.name)
+            log.debug('staffgrp=%s' % group.name)
+            user.groups.add(group)
+
+    elif action=='Remove course staff':
+        uname = request.POST['staffuser']
+        try:
+            user = User.objects.get(username=uname)
+        except User.DoesNotExist:
+            msg += '<font color="red">Error: unknown username "%s"</font>' % uname
+            user = None
+        if user is not None:
+            group = get_staff_group(course)
+            msg += '<font color="green">Removed %s from staff group = %s</font>' % (user,group.name)
+            log.debug('staffgrp=%s' % group.name)
+            user.groups.remove(group)
+
+    # For now, mostly a static page
+    context = {'course': course,
+               'staff_access': True,
+               'admin_access' : request.user.is_staff,
+               'instructor_access' : instructor_access,
+               'datatable' : datatable,
+               'msg' : msg,
+               }
+
+    return render_to_response('courseware/instructor_dashboard.html', context)
+
+def get_student_grade_summary_data(request, course, course_id, get_grades=True, get_raw_scores=False):
+    '''
+    Return data arrays with student identity and grades for specified course.
+
+    course = CourseDescriptor
+    course_id = course ID
+
+    Note: both are passed in, only because instructor_dashboard already has them already.
+    
+    returns datatable = dict(header=header, data=data)
+    where
+
+    header = list of strings labeling the data fields
+    data = list (one per student) of lists of data corresponding to the fields
+
+    If get_raw_scores=True, then instead of grade summaries, the raw grades for all graded modules are returned.
+
+    '''
+    enrolled_students = User.objects.filter(courseenrollment__course_id=course_id).order_by('username')
+
+    header = ['ID', 'Username','Full Name','edX email','External email']
+    if get_grades:
+        gradeset = grades.grade(enrolled_students[0], request, course, keep_raw_scores=get_raw_scores)	# just to construct the header
+        # log.debug('student %s gradeset %s' % (enrolled_students[0], gradeset))
+        if get_raw_scores:
+            header += [score.section for score in gradeset['raw_scores']]
+        else:
+            header += [x['label'] for x in gradeset['section_breakdown']]
+
+    datatable = {'header': header}
+    data = []
+
+    for student in enrolled_students:
+        datarow  = [ student.id, student.username, student.profile.name, student.email ]
+        try:
+            datarow.append(student.externalauthmap.external_email)
+        except:	# ExternalAuthMap.DoesNotExist
+            datarow.append('')
+
+        if get_grades:
+            gradeset = grades.grade(student, request, course, keep_raw_scores=get_raw_scores)
+            # log.debug('student=%s, gradeset=%s' % (student,gradeset))
+            if get_raw_scores:
+                datarow += [score.earned for score in gradeset['raw_scores']]
+            else:
+                datarow += [x['percent'] for x in gradeset['section_breakdown']]
+
+        data.append(datarow)
+    datatable['data'] = data
+    return datatable
+
+@cache_control(no_cache=True, no_store=True, must_revalidate=True)
+def gradebook(request, course_id):
+    """
+    Show the gradebook for this course:
+    - only displayed to course staff
+    - shows students who are enrolled.
+    """
+    course = get_course_with_access(request.user, course_id, 'staff')
+
+    enrolled_students = User.objects.filter(courseenrollment__course_id=course_id).order_by('username')
+
+    # TODO (vshnayder): implement pagination.
+    enrolled_students = enrolled_students[:1000]   # HACK!
+
+    student_info = [{'username': student.username,
+                     'id': student.id,
+                     'email': student.email,
+                     'grade_summary': grades.grade(student, request, course),
+                     'realname': student.profile.name,
+                     }
+                     for student in enrolled_students]
+
+    return render_to_response('courseware/gradebook.html', {'students': student_info,
+                                                 'course': course,
+                                                 'course_id': course_id,
+                                                 # Checked above
+                                                 'staff_access': True,})
+
+
+@cache_control(no_cache=True, no_store=True, must_revalidate=True)
+def grade_summary(request, course_id):
+    """Display the grade summary for a course."""
+    course = get_course_with_access(request.user, course_id, 'staff')
+
+    # For now, just a static page
+    context = {'course': course,
+               'staff_access': True,}
+    return render_to_response('courseware/grade_summary.html', context)
+
+@ensure_csrf_cookie
+@cache_control(no_cache=True, no_store=True, must_revalidate=True)
+def enroll_students(request, course_id):
+    ''' Allows a staff member to enroll students in a course.
+
+    This is a short-term hack for Berkeley courses launching fall
+    2012. In the long term, we would like functionality like this, but
+    we would like both the instructor and the student to agree. Right
+    now, this allows any instructor to add students to their course,
+    which we do not want.
+
+    It is poorly written and poorly tested, but it's designed to be
+    stripped out.
+    '''
+
+    course = get_course_with_access(request.user, course_id, 'staff')
+    existing_students = [ce.user.email for ce in CourseEnrollment.objects.filter(course_id = course_id)]
+
+    if 'new_students' in request.POST:
+        new_students = request.POST['new_students'].split('\n')
+    else:
+        new_students = []
+    new_students = [s.strip() for s in new_students]
+
+    added_students = []
+    rejected_students = []
+
+    for student in new_students:
+        try:
+            nce = CourseEnrollment(user=User.objects.get(email = student), course_id = course_id)
+            nce.save()
+            added_students.append(student)
+        except:
+            rejected_students.append(student)
+
+    return render_to_response("enroll_students.html", {'course':course_id,
+                                                       'existing_students': existing_students,
+                                                       'added_students': added_students,
+                                                       'rejected_students': rejected_students,
+                                                       'debug':new_students})
+
+#-----------------------------------------------------------------------------
+
+def compute_course_stats(course):
+    '''
+    Compute course statistics, including number of problems, videos, html.
+
+    course is a CourseDescriptor from the xmodule system.
+    '''
+
+    # walk the course by using get_children() until we come to the leaves; count the
+    # number of different leaf types
+
+    counts = defaultdict(int)
+
+    print "hello world"
+
+    def walk(module):
+        children = module.get_children()
+        if not children:
+            category = module.__class__.__name__	# HtmlDescriptor, CapaDescriptor, ...
+            counts[category] += 1
+            return
+        for c in children:
+            # print c.__class__.__name__
+            walk(c)
+
+    walk(course)
+
+    print "course %s counts=%s" % (course.display_name,counts)
+    
+    stats = dict(counts)	# number of each kind of module
+
+    return stats
+
diff --git a/lms/envs/common.py b/lms/envs/common.py
index 5cd28d24d9..cf9a767d9f 100644
--- a/lms/envs/common.py
+++ b/lms/envs/common.py
@@ -604,6 +604,7 @@ INSTALLED_APPS = (
     'track',
     'util',
     'certificates',
+    'instructor',
     
     #For the wiki
     'wiki', # The new django-wiki from benjaoming
diff --git a/lms/templates/courseware/instructor_dashboard.html b/lms/templates/courseware/instructor_dashboard.html
index 9508624f9b..49b16cf122 100644
--- a/lms/templates/courseware/instructor_dashboard.html
+++ b/lms/templates/courseware/instructor_dashboard.html
@@ -8,17 +8,98 @@
 
 <%include file="/courseware/course_navigation.html" args="active_page='instructor'" />
 
+<style type="text/css">
+table.stat_table {
+	font-family: verdana,arial,sans-serif;
+	font-size:11px;
+	color:#333333;
+	border-width: 1px;
+	border-color: #666666;
+	border-collapse: collapse;
+}
+table.stat_table th {
+	border-width: 1px;
+	padding: 8px;
+	border-style: solid;
+	border-color: #666666;
+	background-color: #dedede;
+}
+table.stat_table td {
+	border-width: 1px;
+	padding: 8px;
+	border-style: solid;
+	border-color: #666666;
+	background-color: #ffffff;
+}
+</style>
+
 <section class="container">
 <div class="instructor-dashboard-wrapper">
   <section class="instructor-dashboard-content">
     <h1>Instructor Dashboard</h1>
 
+    <form method="POST">
+    <input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }">
+
     <p>
     <a href="${reverse('gradebook', kwargs=dict(course_id=course.id))}">Gradebook</a>
 
     <p>
     <a href="${reverse('grade_summary', kwargs=dict(course_id=course.id))}">Grade summary</a>
 
+    <p>
+    <input type="submit" name="action" value="Dump list of enrolled students">
+
+    <p>
+    <input type="submit" name="action" value="Dump Grades for all students in this course">
+    <input type="submit" name="action" value="Download CSV of all student grades for this course">
+
+    <p>
+    <input type="submit" name="action" value="Dump all RAW grades for all students in this course">
+    <input type="submit" name="action" value="Download CSV of all RAW grades">
+
+%if instructor_access:
+    <hr width="40%" style="align:left">
+    <p>
+    <input type="submit" name="action" value="List course staff members"> 
+    <p>
+    <input type="text" name="staffuser"> <input type="submit" name="action" value="Remove course staff"> 
+    <input type="submit" name="action" value="Add course staff"> 
+    <hr width="40%" style="align:left">
+ %endif
+ 
+%if admin_access:
+    <p>
+    <input type="submit" name="action" value="Reload course from XML files">
+%endif
+
+    </form>
+
+    <br/>
+    <br/>
+    <p>
+      <hr width="100%">
+      <h2>${datatable['title']}</h2>
+      <table class="stat_table">
+        <tr>
+	%for hname in datatable['header']:
+	  <th>${hname}</th>
+	%endfor
+	</tr>
+        %for row in datatable['data']:
+          <tr>
+	    %for value in row:
+	      <td>${value}</td>
+	    %endfor
+	  </tr>
+        %endfor
+      </table>
+    </p>
+
+%if msg:
+    <p>${msg}</p>    
+%endif
+
   </section>
 </div>
 </section>
diff --git a/lms/urls.py b/lms/urls.py
index 278239751b..26aa10a3f4 100644
--- a/lms/urls.py
+++ b/lms/urls.py
@@ -153,14 +153,14 @@ if settings.COURSEWARE_ENABLED:
 
         # For the instructor
         url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/instructor$',
-            'courseware.views.instructor_dashboard', name="instructor_dashboard"),
+            'instructor.views.instructor_dashboard', name="instructor_dashboard"),
 
         url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/gradebook$',
-            'courseware.views.gradebook', name='gradebook'),
+            'instructor.views.gradebook', name='gradebook'),
         url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/grade_summary$',
-            'courseware.views.grade_summary', name='grade_summary'),
+            'instructor.views.grade_summary', name='grade_summary'),
         url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/enroll_students$',
-            'courseware.views.enroll_students', name='enroll_students'),
+            'instructor.views.enroll_students', name='enroll_students'),
     )
 
     # discussion forums live within courseware, so courseware must be enabled first

From 4d5ca284b2c798c6d3c0d1e71d05ae649e5dbc29 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Sun, 2 Sep 2012 19:10:47 -0400
Subject: [PATCH 148/181] add fix to histogram (so 6.00x codemirror is not
 broken)

---
 common/djangoapps/xmodule_modifiers.py |   2 +-
 rakefile                               | 229 -------------------------
 2 files changed, 1 insertion(+), 230 deletions(-)
 delete mode 100644 rakefile

diff --git a/common/djangoapps/xmodule_modifiers.py b/common/djangoapps/xmodule_modifiers.py
index 10cce1a95b..6c3c237ea6 100644
--- a/common/djangoapps/xmodule_modifiers.py
+++ b/common/djangoapps/xmodule_modifiers.py
@@ -76,7 +76,7 @@ def grade_histogram(module_id):
 
     grades = list(cursor.fetchall())
     grades.sort(key=lambda x: x[0])          # Add ORDER BY to sql query?
-    if len(grades) == 1 and grades[0][0] is None:
+    if len(grades) >= 1 and grades[0][0] is None:
         return []
     return grades
 
diff --git a/rakefile b/rakefile
deleted file mode 100644
index 053abf56a8..0000000000
--- a/rakefile
+++ /dev/null
@@ -1,229 +0,0 @@
-require 'rake/clean'
-require 'tempfile'
-
-# Build Constants
-REPO_ROOT = File.dirname(__FILE__)
-BUILD_DIR = File.join(REPO_ROOT, "build")
-REPORT_DIR = File.join(REPO_ROOT, "reports")
-LMS_REPORT_DIR = File.join(REPORT_DIR, "lms")
-
-# Packaging constants
-DEPLOY_DIR = "/opt/wwc"
-PACKAGE_NAME = "mitx"
-LINK_PATH = "/opt/wwc/mitx"
-PKG_VERSION = "0.1"
-COMMIT = (ENV["GIT_COMMIT"] || `git rev-parse HEAD`).chomp()[0, 10]
-BRANCH = (ENV["GIT_BRANCH"] || `git symbolic-ref -q HEAD`).chomp().gsub('refs/heads/', '').gsub('origin/', '')
-BUILD_NUMBER = (ENV["BUILD_NUMBER"] || "dev").chomp()
-
-if BRANCH == "master"
-    DEPLOY_NAME = "#{PACKAGE_NAME}-#{BUILD_NUMBER}-#{COMMIT}"
-else
-    DEPLOY_NAME = "#{PACKAGE_NAME}-#{BRANCH}-#{BUILD_NUMBER}-#{COMMIT}"
-end
-PACKAGE_REPO = "packages@gp.mitx.mit.edu:/opt/pkgrepo.incoming"
-
-NORMALIZED_DEPLOY_NAME = DEPLOY_NAME.downcase().gsub(/[_\/]/, '-')
-INSTALL_DIR_PATH = File.join(DEPLOY_DIR, NORMALIZED_DEPLOY_NAME)
-# Set up the clean and clobber tasks
-CLOBBER.include(BUILD_DIR, REPORT_DIR, 'cover*', '.coverage', 'test_root/*_repo', 'test_root/staticfiles')
-CLEAN.include("#{BUILD_DIR}/*.deb", "#{BUILD_DIR}/util")
-
-def select_executable(*cmds)
-    cmds.find_all{ |cmd| system("which #{cmd} > /dev/null 2>&1") }[0] || fail("No executables found from #{cmds.join(', ')}")
-end
-
-def django_admin(system, env, command, *args)
-    django_admin = ENV['DJANGO_ADMIN_PATH'] || select_executable('django-admin.py', 'django-admin')
-    return "#{django_admin} #{command} --settings=#{system}.envs.#{env} --pythonpath=. #{args.join(' ')}"
-end
-
-task :default => [:test, :pep8, :pylint]
-
-directory REPORT_DIR
-
-default_options = {
-    :lms => '8000',
-    :cms => '8001',
-}
-
-task :predjango do
-    sh("find . -type f -name *.pyc -delete")
-    sh('pip install -e common/lib/xmodule')
-    sh('git submodule update --init')
-end
-
-task :clean_test_files do
-    sh("git clean -fdx test_root")
-end
-
-[:lms, :cms, :common].each do |system|
-    report_dir = File.join(REPORT_DIR, system.to_s)
-    directory report_dir
-
-    desc "Run pep8 on all #{system} code"
-    task "pep8_#{system}" => report_dir do
-        sh("pep8 --ignore=E501 #{system}/djangoapps #{system}/lib | tee #{report_dir}/pep8.report")
-    end
-    task :pep8 => "pep8_#{system}"
-
-    desc "Run pylint on all #{system} code"
-    task "pylint_#{system}" => report_dir do
-        Dir["#{system}/djangoapps/*", "#{system}/lib/*"].each do |app|
-            if File.exists? "#{app}/setup.py"
-                pythonpath_prefix = "PYTHONPATH=#{app}"
-            else
-                pythonpath_prefix = "PYTHONPATH=#{File.dirname(app)}"
-            end
-            app = File.basename(app)
-            sh("#{pythonpath_prefix} pylint --rcfile=.pylintrc -f parseable #{app} | tee #{report_dir}/#{app}.pylint.report")
-        end
-    end
-    task :pylint => "pylint_#{system}"
-end
-
-$failed_tests = 0
-
-def run_tests(system, report_dir, stop_on_failure=true)
-    ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
-    ENV['NOSE_COVER_HTML_DIR'] = File.join(report_dir, "cover")
-    dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"]
-    sh(django_admin(system, :test, 'test', *dirs.each)) do |ok, res|
-        if !ok and stop_on_failure
-            abort "Test failed!"
-        end
-        $failed_tests += 1 unless ok
-    end
-end
-
-TEST_TASKS = []
-
-[:lms, :cms].each do |system|
-    report_dir = File.join(REPORT_DIR, system.to_s)
-    directory report_dir
-
-    # Per System tasks
-    desc "Run all django tests on our djangoapps for the #{system}"
-    task "test_#{system}", [:stop_on_failure] => ["clean_test_files", "#{system}:collectstatic:test", "fasttest_#{system}"]
-
-    # Have a way to run the tests without running collectstatic -- useful when debugging without
-    # messing with static files.
-    task "fasttest_#{system}", [:stop_on_failure] => [report_dir, :predjango] do |t, args|
-        args.with_defaults(:stop_on_failure => 'true')
-        run_tests(system, report_dir, args.stop_on_failure)
-    end
-
-    TEST_TASKS << "test_#{system}"
-
-    desc <<-desc
-        Start the #{system} locally with the specified environment (defaults to dev).
-        Other useful environments are devplus (for dev testing with a real local database)
-        desc
-    task system, [:env, :options] => [:predjango] do |t, args|
-        args.with_defaults(:env => 'dev', :options => default_options[system])
-        sh(django_admin(system, args.env, 'runserver', args.options))
-    end
-
-    # Per environment tasks
-    Dir["#{system}/envs/*.py"].each do |env_file|
-        env = File.basename(env_file).gsub(/\.py/, '')
-        desc "Attempt to import the settings file #{system}.envs.#{env} and report any errors"
-        task "#{system}:check_settings:#{env}" => :predjango do
-            sh("echo 'import #{system}.envs.#{env}' | #{django_admin(system, env, 'shell')}")
-        end
-
-        desc "Run collectstatic in the specified environment"
-        task "#{system}:collectstatic:#{env}" => :predjango do
-            sh("#{django_admin(system, env, 'collectstatic', '--noinput')}")
-        end
-    end
-end
-
-Dir["common/lib/*"].each do |lib|
-    task_name = "test_#{lib}"
-
-    report_dir = File.join(REPORT_DIR, task_name.gsub('/', '_'))
-    directory report_dir
-
-    desc "Run tests for common lib #{lib}"
-    task task_name => report_dir do
-        ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
-        sh("nosetests #{lib} --cover-erase --with-xunit --with-xcoverage --cover-html --cover-inclusive --cover-package #{File.basename(lib)} --cover-html-dir #{File.join(report_dir, "cover")}")
-    end
-    TEST_TASKS << task_name
-end
-
-task :test do
-    TEST_TASKS.each do |task|
-        Rake::Task[task].invoke(false)
-    end
-
-    if $failed_tests > 0
-        abort "Tests failed!"
-    end
-end
-
-task :runserver => :lms
-
-desc "Run django-admin <action> against the specified system and environment"
-task "django-admin", [:action, :system, :env, :options] => [:predjango] do |t, args|
-    args.with_defaults(:env => 'dev', :system => 'lms', :options => '')
-    sh(django_admin(args.system, args.env, args.action, args.options))
-end
-
-task :package do
-    FileUtils.mkdir_p(BUILD_DIR)
-
-    Dir.chdir(BUILD_DIR) do
-        afterremove = Tempfile.new('afterremove')
-        afterremove.write <<-AFTERREMOVE.gsub(/^\s*/, '')
-        #! /bin/bash
-        set -e
-        set -x
-
-        # to be a little safer this rm is executed
-        # as the makeitso user
-
-        if [[ -d "#{INSTALL_DIR_PATH}" ]]; then
-            sudo rm -rf "#{INSTALL_DIR_PATH}"
-        fi
-
-        AFTERREMOVE
-        afterremove.close()
-        FileUtils.chmod(0755, afterremove.path)
-
-        args = ["fakeroot", "fpm", "-s", "dir", "-t", "deb",
-            "--after-remove=#{afterremove.path}",
-            "--prefix=#{INSTALL_DIR_PATH}",
-            "--exclude=**/build/**",
-            "--exclude=**/rakefile",
-            "--exclude=**/.git/**",
-            "--exclude=**/*.pyc",
-            "--exclude=**/reports/**",
-            "--exclude=**/test_root/**",
-            "--exclude=**/.coverage/**",
-            "-C", "#{REPO_ROOT}",
-            "--provides=#{PACKAGE_NAME}",
-            "--name=#{NORMALIZED_DEPLOY_NAME}",
-            "--version=#{PKG_VERSION}",
-            "-a", "all",
-            "."]
-        system(*args) || raise("fpm failed to build the .deb")
-    end
-end
-
-task :publish => :package do
-    sh("scp #{BUILD_DIR}/#{NORMALIZED_DEPLOY_NAME}_#{PKG_VERSION}*.deb #{PACKAGE_REPO}")
-end
-
-namespace :cms do
-  desc "Import course data within the given DATA_DIR variable"
-  task :import do
-    if ENV['DATA_DIR']
-      sh(django_admin(:cms, :dev, :import, ENV['DATA_DIR']))
-    else
-      raise "Please specify a DATA_DIR variable that point to your data directory.\n" +
-        "Example: \`rake cms:import DATA_DIR=../data\`"
-    end
-  end
-end

From 0ddd8949ba856973edffe17084b2d1336d87b29c Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Sun, 2 Sep 2012 19:17:19 -0400
Subject: [PATCH 149/181] re-add rakefile which was deleted by mistake

---
 rakefile | 229 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 229 insertions(+)
 create mode 100644 rakefile

diff --git a/rakefile b/rakefile
new file mode 100644
index 0000000000..053abf56a8
--- /dev/null
+++ b/rakefile
@@ -0,0 +1,229 @@
+require 'rake/clean'
+require 'tempfile'
+
+# Build Constants
+REPO_ROOT = File.dirname(__FILE__)
+BUILD_DIR = File.join(REPO_ROOT, "build")
+REPORT_DIR = File.join(REPO_ROOT, "reports")
+LMS_REPORT_DIR = File.join(REPORT_DIR, "lms")
+
+# Packaging constants
+DEPLOY_DIR = "/opt/wwc"
+PACKAGE_NAME = "mitx"
+LINK_PATH = "/opt/wwc/mitx"
+PKG_VERSION = "0.1"
+COMMIT = (ENV["GIT_COMMIT"] || `git rev-parse HEAD`).chomp()[0, 10]
+BRANCH = (ENV["GIT_BRANCH"] || `git symbolic-ref -q HEAD`).chomp().gsub('refs/heads/', '').gsub('origin/', '')
+BUILD_NUMBER = (ENV["BUILD_NUMBER"] || "dev").chomp()
+
+if BRANCH == "master"
+    DEPLOY_NAME = "#{PACKAGE_NAME}-#{BUILD_NUMBER}-#{COMMIT}"
+else
+    DEPLOY_NAME = "#{PACKAGE_NAME}-#{BRANCH}-#{BUILD_NUMBER}-#{COMMIT}"
+end
+PACKAGE_REPO = "packages@gp.mitx.mit.edu:/opt/pkgrepo.incoming"
+
+NORMALIZED_DEPLOY_NAME = DEPLOY_NAME.downcase().gsub(/[_\/]/, '-')
+INSTALL_DIR_PATH = File.join(DEPLOY_DIR, NORMALIZED_DEPLOY_NAME)
+# Set up the clean and clobber tasks
+CLOBBER.include(BUILD_DIR, REPORT_DIR, 'cover*', '.coverage', 'test_root/*_repo', 'test_root/staticfiles')
+CLEAN.include("#{BUILD_DIR}/*.deb", "#{BUILD_DIR}/util")
+
+def select_executable(*cmds)
+    cmds.find_all{ |cmd| system("which #{cmd} > /dev/null 2>&1") }[0] || fail("No executables found from #{cmds.join(', ')}")
+end
+
+def django_admin(system, env, command, *args)
+    django_admin = ENV['DJANGO_ADMIN_PATH'] || select_executable('django-admin.py', 'django-admin')
+    return "#{django_admin} #{command} --settings=#{system}.envs.#{env} --pythonpath=. #{args.join(' ')}"
+end
+
+task :default => [:test, :pep8, :pylint]
+
+directory REPORT_DIR
+
+default_options = {
+    :lms => '8000',
+    :cms => '8001',
+}
+
+task :predjango do
+    sh("find . -type f -name *.pyc -delete")
+    sh('pip install -e common/lib/xmodule')
+    sh('git submodule update --init')
+end
+
+task :clean_test_files do
+    sh("git clean -fdx test_root")
+end
+
+[:lms, :cms, :common].each do |system|
+    report_dir = File.join(REPORT_DIR, system.to_s)
+    directory report_dir
+
+    desc "Run pep8 on all #{system} code"
+    task "pep8_#{system}" => report_dir do
+        sh("pep8 --ignore=E501 #{system}/djangoapps #{system}/lib | tee #{report_dir}/pep8.report")
+    end
+    task :pep8 => "pep8_#{system}"
+
+    desc "Run pylint on all #{system} code"
+    task "pylint_#{system}" => report_dir do
+        Dir["#{system}/djangoapps/*", "#{system}/lib/*"].each do |app|
+            if File.exists? "#{app}/setup.py"
+                pythonpath_prefix = "PYTHONPATH=#{app}"
+            else
+                pythonpath_prefix = "PYTHONPATH=#{File.dirname(app)}"
+            end
+            app = File.basename(app)
+            sh("#{pythonpath_prefix} pylint --rcfile=.pylintrc -f parseable #{app} | tee #{report_dir}/#{app}.pylint.report")
+        end
+    end
+    task :pylint => "pylint_#{system}"
+end
+
+$failed_tests = 0
+
+def run_tests(system, report_dir, stop_on_failure=true)
+    ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
+    ENV['NOSE_COVER_HTML_DIR'] = File.join(report_dir, "cover")
+    dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"]
+    sh(django_admin(system, :test, 'test', *dirs.each)) do |ok, res|
+        if !ok and stop_on_failure
+            abort "Test failed!"
+        end
+        $failed_tests += 1 unless ok
+    end
+end
+
+TEST_TASKS = []
+
+[:lms, :cms].each do |system|
+    report_dir = File.join(REPORT_DIR, system.to_s)
+    directory report_dir
+
+    # Per System tasks
+    desc "Run all django tests on our djangoapps for the #{system}"
+    task "test_#{system}", [:stop_on_failure] => ["clean_test_files", "#{system}:collectstatic:test", "fasttest_#{system}"]
+
+    # Have a way to run the tests without running collectstatic -- useful when debugging without
+    # messing with static files.
+    task "fasttest_#{system}", [:stop_on_failure] => [report_dir, :predjango] do |t, args|
+        args.with_defaults(:stop_on_failure => 'true')
+        run_tests(system, report_dir, args.stop_on_failure)
+    end
+
+    TEST_TASKS << "test_#{system}"
+
+    desc <<-desc
+        Start the #{system} locally with the specified environment (defaults to dev).
+        Other useful environments are devplus (for dev testing with a real local database)
+        desc
+    task system, [:env, :options] => [:predjango] do |t, args|
+        args.with_defaults(:env => 'dev', :options => default_options[system])
+        sh(django_admin(system, args.env, 'runserver', args.options))
+    end
+
+    # Per environment tasks
+    Dir["#{system}/envs/*.py"].each do |env_file|
+        env = File.basename(env_file).gsub(/\.py/, '')
+        desc "Attempt to import the settings file #{system}.envs.#{env} and report any errors"
+        task "#{system}:check_settings:#{env}" => :predjango do
+            sh("echo 'import #{system}.envs.#{env}' | #{django_admin(system, env, 'shell')}")
+        end
+
+        desc "Run collectstatic in the specified environment"
+        task "#{system}:collectstatic:#{env}" => :predjango do
+            sh("#{django_admin(system, env, 'collectstatic', '--noinput')}")
+        end
+    end
+end
+
+Dir["common/lib/*"].each do |lib|
+    task_name = "test_#{lib}"
+
+    report_dir = File.join(REPORT_DIR, task_name.gsub('/', '_'))
+    directory report_dir
+
+    desc "Run tests for common lib #{lib}"
+    task task_name => report_dir do
+        ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
+        sh("nosetests #{lib} --cover-erase --with-xunit --with-xcoverage --cover-html --cover-inclusive --cover-package #{File.basename(lib)} --cover-html-dir #{File.join(report_dir, "cover")}")
+    end
+    TEST_TASKS << task_name
+end
+
+task :test do
+    TEST_TASKS.each do |task|
+        Rake::Task[task].invoke(false)
+    end
+
+    if $failed_tests > 0
+        abort "Tests failed!"
+    end
+end
+
+task :runserver => :lms
+
+desc "Run django-admin <action> against the specified system and environment"
+task "django-admin", [:action, :system, :env, :options] => [:predjango] do |t, args|
+    args.with_defaults(:env => 'dev', :system => 'lms', :options => '')
+    sh(django_admin(args.system, args.env, args.action, args.options))
+end
+
+task :package do
+    FileUtils.mkdir_p(BUILD_DIR)
+
+    Dir.chdir(BUILD_DIR) do
+        afterremove = Tempfile.new('afterremove')
+        afterremove.write <<-AFTERREMOVE.gsub(/^\s*/, '')
+        #! /bin/bash
+        set -e
+        set -x
+
+        # to be a little safer this rm is executed
+        # as the makeitso user
+
+        if [[ -d "#{INSTALL_DIR_PATH}" ]]; then
+            sudo rm -rf "#{INSTALL_DIR_PATH}"
+        fi
+
+        AFTERREMOVE
+        afterremove.close()
+        FileUtils.chmod(0755, afterremove.path)
+
+        args = ["fakeroot", "fpm", "-s", "dir", "-t", "deb",
+            "--after-remove=#{afterremove.path}",
+            "--prefix=#{INSTALL_DIR_PATH}",
+            "--exclude=**/build/**",
+            "--exclude=**/rakefile",
+            "--exclude=**/.git/**",
+            "--exclude=**/*.pyc",
+            "--exclude=**/reports/**",
+            "--exclude=**/test_root/**",
+            "--exclude=**/.coverage/**",
+            "-C", "#{REPO_ROOT}",
+            "--provides=#{PACKAGE_NAME}",
+            "--name=#{NORMALIZED_DEPLOY_NAME}",
+            "--version=#{PKG_VERSION}",
+            "-a", "all",
+            "."]
+        system(*args) || raise("fpm failed to build the .deb")
+    end
+end
+
+task :publish => :package do
+    sh("scp #{BUILD_DIR}/#{NORMALIZED_DEPLOY_NAME}_#{PKG_VERSION}*.deb #{PACKAGE_REPO}")
+end
+
+namespace :cms do
+  desc "Import course data within the given DATA_DIR variable"
+  task :import do
+    if ENV['DATA_DIR']
+      sh(django_admin(:cms, :dev, :import, ENV['DATA_DIR']))
+    else
+      raise "Please specify a DATA_DIR variable that point to your data directory.\n" +
+        "Example: \`rake cms:import DATA_DIR=../data\`"
+    end
+  end
+end

From 9e8ec09b4d9855c140dfa8b097d2906c48c25e27 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Sun, 2 Sep 2012 20:15:57 -0400
Subject: [PATCH 150/181] pep8 fixes

---
 common/djangoapps/student/models.py | 36 +++++++++----
 lms/djangoapps/instructor/views.py  | 78 ++++++++++++++---------------
 2 files changed, 65 insertions(+), 49 deletions(-)

diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py
index c494035104..529a6f1298 100644
--- a/common/djangoapps/student/models.py
+++ b/common/djangoapps/student/models.py
@@ -4,7 +4,7 @@ Models for Student Information
 Replication Notes
 
 In our live deployment, we intend to run in a scenario where there is a pool of
-Portal servers that hold the canoncial user information and that user 
+Portal servers that hold the canoncial user information and that user
 information is replicated to slave Course server pools. Each Course has a set of
 servers that serves only its content and has users that are relevant only to it.
 
@@ -61,6 +61,7 @@ from xmodule.modulestore.django import modulestore
 
 log = logging.getLogger(__name__)
 
+
 class UserProfile(models.Model):
     """This is where we store all the user demographic fields. We have a 
     separate table for this rather than extending the built-in Django auth_user.
@@ -175,6 +176,7 @@ class PendingEmailChange(models.Model):
     new_email = models.CharField(blank=True, max_length=255, db_index=True)
     activation_key = models.CharField(('activation key'), max_length=32, unique=True, db_index=True)
 
+
 class CourseEnrollment(models.Model):
     user = models.ForeignKey(User)
     course_id = models.CharField(max_length=255, db_index=True)
@@ -185,7 +187,8 @@ class CourseEnrollment(models.Model):
         unique_together = (('user', 'course_id'), )
 
     def __unicode__(self):
-        return "%s: %s (%s)" % (self.user,self.course_id,self.created)
+        return "%s: %s (%s)" % (self.user, self.course_id, self.created)
+
 
 @receiver(post_save, sender=CourseEnrollment)
 def assign_default_role(sender, instance, **kwargs):
@@ -276,6 +279,7 @@ def add_user_to_default_group(user, group):
     utg.users.add(User.objects.get(username=user))
     utg.save()
 
+
 @receiver(post_save, sender=User)
 def update_user_information(sender, instance, created, **kwargs):
     try:
@@ -286,6 +290,7 @@ def update_user_information(sender, instance, created, **kwargs):
         log.error(unicode(e))
         log.error("update user info to discussion failed for user with id: " + str(instance.id))
 
+
 ########################## REPLICATION SIGNALS #################################
 # @receiver(post_save, sender=User)
 def replicate_user_save(sender, **kwargs):
@@ -295,6 +300,7 @@ def replicate_user_save(sender, **kwargs):
     for course_db_name in db_names_to_replicate_to(user_obj.id):
         replicate_user(user_obj, course_db_name)
 
+
 # @receiver(post_save, sender=CourseEnrollment)
 def replicate_enrollment_save(sender, **kwargs):
     """This is called when a Student enrolls in a course. It has to do the 
@@ -320,12 +326,14 @@ def replicate_enrollment_save(sender, **kwargs):
     log.debug("Replicating user profile because of new enrollment")
     user_profile = UserProfile.objects.get(user_id=enrollment_obj.user_id)
     replicate_model(UserProfile.save, user_profile, enrollment_obj.user_id)
- 
+
+
 # @receiver(post_delete, sender=CourseEnrollment)
 def replicate_enrollment_delete(sender, **kwargs):
     enrollment_obj = kwargs['instance']
     return replicate_model(CourseEnrollment.delete, enrollment_obj, enrollment_obj.user_id)
- 
+
+
 # @receiver(post_save, sender=UserProfile)
 def replicate_userprofile_save(sender, **kwargs):
     """We just updated the UserProfile (say an update to the name), so push that
@@ -333,12 +341,13 @@ def replicate_userprofile_save(sender, **kwargs):
     user_profile_obj = kwargs['instance']
     return replicate_model(UserProfile.save, user_profile_obj, user_profile_obj.user_id)
 
- 
+
 ######### Replication functions #########
 USER_FIELDS_TO_COPY = ["id", "username", "first_name", "last_name", "email",
                        "password", "is_staff", "is_active", "is_superuser",
                        "last_login", "date_joined"]
 
+
 def replicate_user(portal_user, course_db_name):
     """Replicate a User to the correct Course DB. This is more complicated than
     it should be because Askbot extends the auth_user table and adds its own 
@@ -362,9 +371,10 @@ def replicate_user(portal_user, course_db_name):
     course_user.save(using=course_db_name)
     unmark(course_user)
 
+
 def replicate_model(model_method, instance, user_id):
     """
-    model_method is the model action that we want replicated. For instance, 
+    model_method is the model action that we want replicated. For instance,
                  UserProfile.save
     """
     if not should_replicate(instance):
@@ -379,8 +389,10 @@ def replicate_model(model_method, instance, user_id):
         model_method(instance, using=db_name)
     unmark(instance)
 
+
 ######### Replication Helpers #########
 
+
 def is_valid_course_id(course_id):
     """Right now, the only database that's not a course database is 'default'.
     I had nicer checking in here originally -- it would scan the courses that
@@ -390,26 +402,30 @@ def is_valid_course_id(course_id):
     """
     return course_id != 'default'
 
+
 def is_portal():
     """Are we in the portal pool? Only Portal servers are allowed to replicate
     their changes. For now, only Portal servers see multiple DBs, so we use
     that to decide."""
     return len(settings.DATABASES) > 1
 
+
 def db_names_to_replicate_to(user_id):
     """Return a list of DB names that this user_id is enrolled in."""
     return [c.course_id
             for c in CourseEnrollment.objects.filter(user_id=user_id)
             if is_valid_course_id(c.course_id)]
 
+
 def marked_handled(instance):
     """Have we marked this instance as being handled to avoid infinite loops
     caused by saving models in post_save hooks for the same models?"""
     return hasattr(instance, '_do_not_copy_to_course_db') and instance._do_not_copy_to_course_db
 
+
 def mark_handled(instance):
     """You have to mark your instance with this function or else we'll go into
-    an infinite loop since we're putting listeners on Model saves/deletes and 
+    an infinite loop since we're putting listeners on Model saves/deletes and
     the act of replication requires us to call the same model method.
 
     We create a _replicated attribute to differentiate the first save of this
@@ -418,16 +434,18 @@ def mark_handled(instance):
     """
     instance._do_not_copy_to_course_db = True
 
+
 def unmark(instance):
-    """If we don't unmark a model after we do replication, then consecutive 
+    """If we don't unmark a model after we do replication, then consecutive
     save() calls won't be properly replicated."""
     instance._do_not_copy_to_course_db = False
 
+
 def should_replicate(instance):
     """Should this instance be replicated? We need to be a Portal server and
     the instance has to not have been marked_handled."""
     if marked_handled(instance):
-        # Basically, avoid an infinite loop. You should 
+        # Basically, avoid an infinite loop. You should
         log.debug("{0} should not be replicated because it's been marked"
                   .format(instance))
         return False
diff --git a/lms/djangoapps/instructor/views.py b/lms/djangoapps/instructor/views.py
index ccf40cd5e5..bfbfab6e25 100644
--- a/lms/djangoapps/instructor/views.py
+++ b/lms/djangoapps/instructor/views.py
@@ -38,54 +38,55 @@ log = logging.getLogger("mitx.courseware")
 
 template_imports = {'urllib': urllib}
 
+
 @ensure_csrf_cookie
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
 def instructor_dashboard(request, course_id):
     """Display the instructor dashboard for a course."""
     course = get_course_with_access(request.user, course_id, 'staff')
 
-    instructor_access = has_access(request.user, course, 'instructor')	# an instructor can manage staff lists
+    instructor_access = has_access(request.user, course, 'instructor')		# an instructor can manage staff lists
 
     msg = ''
     # msg += ('POST=%s' % dict(request.POST)).replace('<','&lt;')
 
     def escape(s):
         """escape HTML special characters in string"""
-        return str(s).replace('<','&lt;').replace('>','&gt;')
+        return str(s).replace('<', '&lt;').replace('>', '&gt;')
 
     # assemble some course statistics for output to instructor
-    datatable = {'header': ['Statistic','Value'],
+    datatable = {'header': ['Statistic', 'Value'],
                  'title': 'Course Statistics At A Glance',
                  }
-    data = [ ['# Enrolled' ,CourseEnrollment.objects.filter(course_id=course_id).count()] ]
+    data = [['# Enrolled', CourseEnrollment.objects.filter(course_id=course_id).count()]]
     data += compute_course_stats(course).items()
     if request.user.is_staff:
         data.append(['metadata', escape(str(course.metadata))])
     datatable['data'] = data
 
-    def return_csv(fn,datatable):
+    def return_csv(fn, datatable):
         response = HttpResponse(mimetype='text/csv')
         response['Content-Disposition'] = 'attachment; filename=%s' % fn
-        writer = csv.writer(response,dialect='excel',quotechar='"', quoting=csv.QUOTE_ALL)
+        writer = csv.writer(response, dialect='excel', quotechar='"', quoting=csv.QUOTE_ALL)
         writer.writerow(datatable['header'])
         for datarow in datatable['data']:
             writer.writerow(datarow)
         return response
 
     def get_staff_group(course):
-        staffgrp = get_access_group_name(course,'staff')
+        staffgrp = get_access_group_name(course, 'staff')
         try:
             group = Group.objects.get(name=staffgrp)
         except Group.DoesNotExist:
-            group = Group(name=staffgrp)	# create the group
+            group = Group(name=staffgrp)		# create the group
             group.save()
         return group
 
     # process actions from form POST
-    action = request.POST.get('action','')
+    action = request.POST.get('action', '')
 
     if 'Reload' in action:
-        log.debug('reloading %s (%s)' % (course_id,course))
+        log.debug('reloading %s (%s)' % (course_id, course))
         try:
             data_dir = course.metadata['data_dir']
             modulestore().try_load_course(data_dir)
@@ -93,7 +94,7 @@ def instructor_dashboard(request, course_id):
         except Exception as err:
             msg += '<br/><p>Error: %s</p>' % escape(err)
 
-    elif action=='Dump list of enrolled students':
+    elif action == 'Dump list of enrolled students':
         log.debug(action)
         datatable = get_student_grade_summary_data(request, course, course_id, get_grades=False)
         datatable['title'] = 'List of students enrolled in %s' % course_id
@@ -122,11 +123,11 @@ def instructor_dashboard(request, course_id):
         msg += 'Staff group = %s' % group.name
         log.debug('staffgrp=%s' % group.name)
         uset = group.user_set.all()
-        datatable = {'header': ['Username','Full name']}
-        datatable['data'] = [[ x.username, x.profile.name ] for x in uset]
+        datatable = {'header': ['Username', 'Full name']}
+        datatable['data'] = [[x.username, x.profile.name] for x in uset]
         datatable['title'] = 'List of Staff in course %s' % course_id
 
-    elif action=='Add course staff':
+    elif action == 'Add course staff':
         uname = request.POST['staffuser']
         try:
             user = User.objects.get(username=uname)
@@ -135,11 +136,11 @@ def instructor_dashboard(request, course_id):
             user = None
         if user is not None:
             group = get_staff_group(course)
-            msg += '<font color="green">Added %s to staff group = %s</font>' % (user,group.name)
+            msg += '<font color="green">Added %s to staff group = %s</font>' % (user, group.name)
             log.debug('staffgrp=%s' % group.name)
             user.groups.add(group)
 
-    elif action=='Remove course staff':
+    elif action == 'Remove course staff':
         uname = request.POST['staffuser']
         try:
             user = User.objects.get(username=uname)
@@ -148,21 +149,22 @@ def instructor_dashboard(request, course_id):
             user = None
         if user is not None:
             group = get_staff_group(course)
-            msg += '<font color="green">Removed %s from staff group = %s</font>' % (user,group.name)
+            msg += '<font color="green">Removed %s from staff group = %s</font>' % (user, group.name)
             log.debug('staffgrp=%s' % group.name)
             user.groups.remove(group)
 
     # For now, mostly a static page
     context = {'course': course,
                'staff_access': True,
-               'admin_access' : request.user.is_staff,
-               'instructor_access' : instructor_access,
-               'datatable' : datatable,
-               'msg' : msg,
+               'admin_access': request.user.is_staff,
+               'instructor_access': instructor_access,
+               'datatable': datatable,
+               'msg': msg,
                }
 
     return render_to_response('courseware/instructor_dashboard.html', context)
 
+
 def get_student_grade_summary_data(request, course, course_id, get_grades=True, get_raw_scores=False):
     '''
     Return data arrays with student identity and grades for specified course.
@@ -171,7 +173,7 @@ def get_student_grade_summary_data(request, course, course_id, get_grades=True,
     course_id = course ID
 
     Note: both are passed in, only because instructor_dashboard already has them already.
-    
+
     returns datatable = dict(header=header, data=data)
     where
 
@@ -183,9 +185,10 @@ def get_student_grade_summary_data(request, course, course_id, get_grades=True,
     '''
     enrolled_students = User.objects.filter(courseenrollment__course_id=course_id).order_by('username')
 
-    header = ['ID', 'Username','Full Name','edX email','External email']
+    header = ['ID', 'Username', 'Full Name', 'edX email', 'External email']
     if get_grades:
-        gradeset = grades.grade(enrolled_students[0], request, course, keep_raw_scores=get_raw_scores)	# just to construct the header
+        # just to construct the header
+        gradeset = grades.grade(enrolled_students[0], request, course, keep_raw_scores=get_raw_scores)
         # log.debug('student %s gradeset %s' % (enrolled_students[0], gradeset))
         if get_raw_scores:
             header += [score.section for score in gradeset['raw_scores']]
@@ -196,7 +199,7 @@ def get_student_grade_summary_data(request, course, course_id, get_grades=True,
     data = []
 
     for student in enrolled_students:
-        datarow  = [ student.id, student.username, student.profile.name, student.email ]
+        datarow = [ student.id, student.username, student.profile.name, student.email ]
         try:
             datarow.append(student.externalauthmap.external_email)
         except:	# ExternalAuthMap.DoesNotExist
@@ -214,6 +217,7 @@ def get_student_grade_summary_data(request, course, course_id, get_grades=True,
     datatable['data'] = data
     return datatable
 
+
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
 def gradebook(request, course_id):
     """
@@ -240,7 +244,7 @@ def gradebook(request, course_id):
                                                  'course': course,
                                                  'course_id': course_id,
                                                  # Checked above
-                                                 'staff_access': True,})
+                                                 'staff_access': True, })
 
 
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
@@ -250,9 +254,10 @@ def grade_summary(request, course_id):
 
     # For now, just a static page
     context = {'course': course,
-               'staff_access': True,}
+               'staff_access': True, }
     return render_to_response('courseware/grade_summary.html', context)
 
+
 @ensure_csrf_cookie
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
 def enroll_students(request, course_id):
@@ -269,7 +274,7 @@ def enroll_students(request, course_id):
     '''
 
     course = get_course_with_access(request.user, course_id, 'staff')
-    existing_students = [ce.user.email for ce in CourseEnrollment.objects.filter(course_id = course_id)]
+    existing_students = [ce.user.email for ce in CourseEnrollment.objects.filter(course_id=course_id)]
 
     if 'new_students' in request.POST:
         new_students = request.POST['new_students'].split('\n')
@@ -282,20 +287,21 @@ def enroll_students(request, course_id):
 
     for student in new_students:
         try:
-            nce = CourseEnrollment(user=User.objects.get(email = student), course_id = course_id)
+            nce = CourseEnrollment(user=User.objects.get(email=student), course_id=course_id)
             nce.save()
             added_students.append(student)
         except:
             rejected_students.append(student)
 
-    return render_to_response("enroll_students.html", {'course':course_id,
+    return render_to_response("enroll_students.html", {'course': course_id,
                                                        'existing_students': existing_students,
                                                        'added_students': added_students,
                                                        'rejected_students': rejected_students,
-                                                       'debug':new_students})
+                                                       'debug': new_students})
 
 #-----------------------------------------------------------------------------
 
+
 def compute_course_stats(course):
     '''
     Compute course statistics, including number of problems, videos, html.
@@ -308,23 +314,15 @@ def compute_course_stats(course):
 
     counts = defaultdict(int)
 
-    print "hello world"
-
     def walk(module):
         children = module.get_children()
         if not children:
-            category = module.__class__.__name__	# HtmlDescriptor, CapaDescriptor, ...
+            category = module.__class__.__name__ 	# HtmlDescriptor, CapaDescriptor, ...
             counts[category] += 1
             return
         for c in children:
-            # print c.__class__.__name__
             walk(c)
 
     walk(course)
-
-    print "course %s counts=%s" % (course.display_name,counts)
-    
     stats = dict(counts)	# number of each kind of module
-
     return stats
-

From 5194dc3177b357d150ee7727d12a271739c4ebc2 Mon Sep 17 00:00:00 2001
From: David Ormsbee <dave@edx.org>
Date: Sun, 2 Sep 2012 22:22:01 -0400
Subject: [PATCH 151/181] Add support for discussion_link attribute in course
 policy files as a kludge to support CS50's external forums

---
 common/lib/xmodule/xmodule/course_module.py     | 9 +++++++++
 lms/templates/courseware/course_navigation.html | 9 ++++++++-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/common/lib/xmodule/xmodule/course_module.py b/common/lib/xmodule/xmodule/course_module.py
index 28668ef088..9ae6fc59b7 100644
--- a/common/lib/xmodule/xmodule/course_module.py
+++ b/common/lib/xmodule/xmodule/course_module.py
@@ -232,6 +232,15 @@ class CourseDescriptor(SequenceDescriptor):
     def info_sidebar_name(self):
         return self.metadata.get('info_sidebar_name', 'Course Handouts')
 
+    @property
+    def discussion_link(self):
+        """FIXME: This is a quick kludge to allow CS50 (and other courses) to 
+        specify their own discussion forums as external links by specifying a 
+        "discussion_link" in their policy JSON file. This should later get
+        folded in with Syllabus, Course Info, and additional Custom tabs in a 
+        more sensible framework later."""
+        return self.metadata.get('discussion_link', None)
+
     @property
     def title(self):
         return self.display_name
diff --git a/lms/templates/courseware/course_navigation.html b/lms/templates/courseware/course_navigation.html
index 92c9632625..f7dac9012e 100644
--- a/lms/templates/courseware/course_navigation.html
+++ b/lms/templates/courseware/course_navigation.html
@@ -28,11 +28,18 @@ def url_class(url):
             <li class="book"><a href="${reverse('book', args=[course.id, index])}" class="${url_class('book')}">${textbook.title}</a></li>
         % endfor
       % endif
-      % if settings.MITX_FEATURES.get('ENABLE_DISCUSSION_SERVICE'):
+
+      ## If they have a discussion link specified, use that even if we feature
+      ## flag discussions off. Disabling that is mostly a server safety feature
+      ## at this point, and we don't need to worry about external sites.
+      % if course.discussion_link:
+        <li class="discussion"><a href="${course.discussion_link}">Discussion</a></li>
+      % elif settings.MITX_FEATURES.get('ENABLE_DISCUSSION_SERVICE'):
         <li class="discussion"><a href="${reverse('django_comment_client.forum.views.forum_form_discussion', args=[course.id])}" class="${url_class('discussion')}">Discussion</a></li>
         ## <li class="news"><a href="${reverse('news', args=[course.id])}" class="${url_class('news')}">News</a></li>
       % endif 
 
+      ## This is Askbot, which we should be retiring soon...
       % if settings.MITX_FEATURES.get('ENABLE_DISCUSSION'):
         <li class="discussion"><a href="${reverse('questions')}">Discussion</a></li>
       % endif

From 4341ea7689ee4570749312c7e80392737f5fca12 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Sun, 2 Sep 2012 22:40:50 -0400
Subject: [PATCH 152/181] fix bug in modulestore reload: etree parser must be
 specified explicitly

---
 common/lib/xmodule/xmodule/xml_module.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py
index 25dc4e0c6e..e8ecfa785a 100644
--- a/common/lib/xmodule/xmodule/xml_module.py
+++ b/common/lib/xmodule/xmodule/xml_module.py
@@ -12,6 +12,9 @@ import sys
 
 log = logging.getLogger(__name__)
 
+edx_xml_parser = etree.XMLParser(dtd_validation=False, load_dtd=False,
+                                 remove_comments=True, remove_blank_text=True)
+
 def name_to_pathname(name):
     """
     Convert a location name for use in a path: replace ':' with '/'.
@@ -146,7 +149,7 @@ class XmlDescriptor(XModuleDescriptor):
 
         Returns an lxml Element
         """
-        return etree.parse(file_object).getroot()
+        return etree.parse(file_object, parser=edx_xml_parser).getroot()
 
     @classmethod
     def load_file(cls, filepath, fs, location):

From 91dcef7fd0fac0f5afcf50418158e4812dee7667 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Sun, 2 Sep 2012 22:41:27 -0400
Subject: [PATCH 153/181] add gitreload to instructor dashboard

---
 lms/djangoapps/instructor/views.py               | 16 ++++++++++++++--
 .../courseware/instructor_dashboard.html         |  3 ++-
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/lms/djangoapps/instructor/views.py b/lms/djangoapps/instructor/views.py
index bfbfab6e25..5a3611285c 100644
--- a/lms/djangoapps/instructor/views.py
+++ b/lms/djangoapps/instructor/views.py
@@ -1,10 +1,11 @@
 # ======== Instructor views =============================================================================
 
 import csv
+import itertools
 import json
 import logging
+import os
 import urllib
-import itertools
 
 from functools import partial
 from collections import defaultdict
@@ -85,7 +86,18 @@ def instructor_dashboard(request, course_id):
     # process actions from form POST
     action = request.POST.get('action', '')
 
-    if 'Reload' in action:
+    if 'GIT pull' in action:
+        data_dir = course.metadata['data_dir']
+        log.debug('git pull %s' % (data_dir))
+        gdir = settings.DATA_DIR / data_dir
+        if not os.path.exists(gdir):
+            msg += "====> ERROR in gitreload - no such directory %s" % gdir
+        else:
+            cmd = "cd %s; git reset --hard HEAD; git clean -f -d; git pull origin; chmod g+w course.xml" % gdir
+            msg += "git pull on %s:<p>" % data_dir
+            msg += "<pre>%s</pre></p>" % escape(os.popen(cmd).read())
+            
+    if 'Reload course' in action:
         log.debug('reloading %s (%s)' % (course_id, course))
         try:
             data_dir = course.metadata['data_dir']
diff --git a/lms/templates/courseware/instructor_dashboard.html b/lms/templates/courseware/instructor_dashboard.html
index 49b16cf122..29397e5c41 100644
--- a/lms/templates/courseware/instructor_dashboard.html
+++ b/lms/templates/courseware/instructor_dashboard.html
@@ -70,7 +70,8 @@ table.stat_table td {
  
 %if admin_access:
     <p>
-    <input type="submit" name="action" value="Reload course from XML files">
+    <input type="submit" name="action" value="Reload course from XML files"> 
+    <input type="submit" name="action" value="GIT pull and Reload course"> 
 %endif
 
     </form>

From 9b056b284d185a48ee2db61de455a960ab7e270e Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Sun, 2 Sep 2012 23:17:45 -0400
Subject: [PATCH 154/181] add tracking logging of instructor dashboard actions

---
 lms/djangoapps/instructor/views.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/lms/djangoapps/instructor/views.py b/lms/djangoapps/instructor/views.py
index 5a3611285c..5a596d3274 100644
--- a/lms/djangoapps/instructor/views.py
+++ b/lms/djangoapps/instructor/views.py
@@ -7,6 +7,8 @@ import logging
 import os
 import urllib
 
+import track.views
+
 from functools import partial
 from collections import defaultdict
 
@@ -96,6 +98,7 @@ def instructor_dashboard(request, course_id):
             cmd = "cd %s; git reset --hard HEAD; git clean -f -d; git pull origin; chmod g+w course.xml" % gdir
             msg += "git pull on %s:<p>" % data_dir
             msg += "<pre>%s</pre></p>" % escape(os.popen(cmd).read())
+            track.views.server_track(request, 'git pull %s' % data_dir, {}, page='idashboard')
             
     if 'Reload course' in action:
         log.debug('reloading %s (%s)' % (course_id, course))
@@ -103,6 +106,7 @@ def instructor_dashboard(request, course_id):
             data_dir = course.metadata['data_dir']
             modulestore().try_load_course(data_dir)
             msg += "<br/><p>Course reloaded from %s</p>" % data_dir
+            track.views.server_track(request, 'reload %s' % data_dir, {}, page='idashboard')
         except Exception as err:
             msg += '<br/><p>Error: %s</p>' % escape(err)
 
@@ -110,23 +114,28 @@ def instructor_dashboard(request, course_id):
         log.debug(action)
         datatable = get_student_grade_summary_data(request, course, course_id, get_grades=False)
         datatable['title'] = 'List of students enrolled in %s' % course_id
+        track.views.server_track(request, 'list-students', {}, page='idashboard')
 
     elif 'Dump Grades' in action:
         log.debug(action)
         datatable = get_student_grade_summary_data(request, course, course_id, get_grades=True)
         datatable['title'] = 'Summary Grades of students enrolled in %s' % course_id
+        track.views.server_track(request, 'dump-grades', {}, page='idashboard')
 
     elif 'Dump all RAW grades' in action:
         log.debug(action)
         datatable = get_student_grade_summary_data(request, course, course_id, get_grades=True,
                                                    get_raw_scores=True)
         datatable['title'] = 'Raw Grades of students enrolled in %s' % course_id
+        track.views.server_track(request, 'dump-grades-raw', {}, page='idashboard')
 
     elif 'Download CSV of all student grades' in action:
+        track.views.server_track(request, 'dump-grades-csv', {}, page='idashboard')
         return return_csv('grades_%s.csv' % course_id,
                           get_student_grade_summary_data(request, course, course_id))
 
     elif 'Download CSV of all RAW grades' in action:
+        track.views.server_track(request, 'dump-grades-csv-raw', {}, page='idashboard')
         return return_csv('grades_%s_raw.csv' % course_id,
                           get_student_grade_summary_data(request, course, course_id, get_raw_scores=True))
 
@@ -138,6 +147,7 @@ def instructor_dashboard(request, course_id):
         datatable = {'header': ['Username', 'Full name']}
         datatable['data'] = [[x.username, x.profile.name] for x in uset]
         datatable['title'] = 'List of Staff in course %s' % course_id
+        track.views.server_track(request, 'list-staff', {}, page='idashboard')
 
     elif action == 'Add course staff':
         uname = request.POST['staffuser']
@@ -151,6 +161,7 @@ def instructor_dashboard(request, course_id):
             msg += '<font color="green">Added %s to staff group = %s</font>' % (user, group.name)
             log.debug('staffgrp=%s' % group.name)
             user.groups.add(group)
+            track.views.server_track(request, 'add-staff %s' % user, {}, page='idashboard')
 
     elif action == 'Remove course staff':
         uname = request.POST['staffuser']
@@ -164,6 +175,7 @@ def instructor_dashboard(request, course_id):
             msg += '<font color="green">Removed %s from staff group = %s</font>' % (user, group.name)
             log.debug('staffgrp=%s' % group.name)
             user.groups.remove(group)
+            track.views.server_track(request, 'remove-staff %s' % user, {}, page='idashboard')
 
     # For now, mostly a static page
     context = {'course': course,

From e50f6229157dfcae76b93df18d0589f17aaf8836 Mon Sep 17 00:00:00 2001
From: Kyle Fiedler <kfiedler@thoughtbot.com>
Date: Mon, 3 Sep 2012 10:40:21 -0400
Subject: [PATCH 155/181] Added styles for detailed solution

---
 .../lib/xmodule/xmodule/css/capa/display.scss | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/common/lib/xmodule/xmodule/css/capa/display.scss b/common/lib/xmodule/xmodule/css/capa/display.scss
index aea59a4d63..fd871b0b03 100644
--- a/common/lib/xmodule/xmodule/css/capa/display.scss
+++ b/common/lib/xmodule/xmodule/css/capa/display.scss
@@ -392,6 +392,28 @@ section.problem {
     }
   }
 
+  .detailed-solution {
+    border: 1px solid #ddd;
+    padding: 9px 9px 20px;
+    margin-bottom: 10px;
+    background: #FFF;
+    position: relative;
+    @include box-shadow(inset 0 0 0 1px #eee);
+    @include border-radius(3px);
+
+    p:first-child {
+      font-size: 0.9em;
+      font-weight: bold;
+      font-style: normal;
+      text-transform: uppercase;
+      color: #AAA;
+    }
+
+    p:last-child {
+      margin-bottom: 0;
+    }
+  }
+
   div.capa_alert {
     padding: 8px 12px;
     border: 1px solid #EBE8BF;

From ea78559ad3603ece4555e3adfba2aaca7fb1117a Mon Sep 17 00:00:00 2001
From: David Ormsbee <dave@edx.org>
Date: Mon, 3 Sep 2012 10:47:15 -0400
Subject: [PATCH 156/181] Changed FIXME -> TODO in response to comments

---
 common/lib/xmodule/xmodule/course_module.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/lib/xmodule/xmodule/course_module.py b/common/lib/xmodule/xmodule/course_module.py
index 9ae6fc59b7..6c8c20014c 100644
--- a/common/lib/xmodule/xmodule/course_module.py
+++ b/common/lib/xmodule/xmodule/course_module.py
@@ -234,7 +234,7 @@ class CourseDescriptor(SequenceDescriptor):
 
     @property
     def discussion_link(self):
-        """FIXME: This is a quick kludge to allow CS50 (and other courses) to 
+        """TODO: This is a quick kludge to allow CS50 (and other courses) to 
         specify their own discussion forums as external links by specifying a 
         "discussion_link" in their policy JSON file. This should later get
         folded in with Syllabus, Course Info, and additional Custom tabs in a 

From 4e2ddd578a1ee16c72740c7ae6a79e371c49adfb Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Mon, 3 Sep 2012 12:32:41 -0400
Subject: [PATCH 157/181] unit tests for instructor dashboard (grades CSV
 download)

---
 lms/djangoapps/instructor/tests.py | 85 ++++++++++++++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 lms/djangoapps/instructor/tests.py

diff --git a/lms/djangoapps/instructor/tests.py b/lms/djangoapps/instructor/tests.py
new file mode 100644
index 0000000000..8b8449dbee
--- /dev/null
+++ b/lms/djangoapps/instructor/tests.py
@@ -0,0 +1,85 @@
+"""
+Unit tests for instructor dashboard
+
+Based on (and depends on) unit tests for courseware.
+
+Notes for running by hand:
+
+django-admin.py test --settings=lms.envs.test --pythonpath=. lms/djangoapps/instructor
+"""
+
+import courseware.tests.tests as ct
+
+from nose import SkipTest
+from mock import patch, Mock
+from override_settings import override_settings
+
+# Need access to internal func to put users in the right group
+from courseware.access import _course_staff_group_name
+from django.contrib.auth.models import User, Group
+from django.conf import settings
+from django.core.urlresolvers import reverse
+
+import xmodule.modulestore.django
+
+from xmodule.modulestore.django import modulestore
+
+
+@override_settings(MODULESTORE=ct.TEST_DATA_XML_MODULESTORE)
+class TestInstructorDashboardGradeDownloadCSV(ct.PageLoader):
+    '''
+    Check for download of csv
+    '''
+
+    def setUp(self):
+        xmodule.modulestore.django._MODULESTORES = {}
+        courses = modulestore().get_courses()
+
+        def find_course(name):
+            """Assumes the course is present"""
+            return [c for c in courses if c.location.course==name][0]
+
+        self.full = find_course("full")
+        self.toy = find_course("toy")
+
+        # Create two accounts
+        self.student = 'view@test.com'
+        self.instructor = 'view2@test.com'
+        self.password = 'foo'
+        self.create_account('u1', self.student, self.password)
+        self.create_account('u2', self.instructor, self.password)
+        self.activate_user(self.student)
+        self.activate_user(self.instructor)
+
+        group_name = _course_staff_group_name(self.toy.location)
+        g = Group.objects.create(name=group_name)
+        g.user_set.add(ct.user(self.instructor))
+
+        self.logout()
+        self.login(self.instructor, self.password)
+        self.enroll(self.toy)
+
+
+    def test_download_grades_csv(self):
+        print "running test_download_grades_csv"
+        course = self.toy
+        url = reverse('instructor_dashboard', kwargs={'course_id': course.id})
+        msg = "url = %s\n" % url
+        resp = self.client.post(url, {
+            'action': 'Download CSV of all student grades for this course',
+            })
+        msg += "instructor dashboard download csv grades: resp = '%s'" % resp
+
+        respstr = str(resp).replace('\r','')
+        #open('idtest.out','w').write(respstr)
+
+        expected_resp = '''Vary: Cookie
+Content-Type: text/csv
+Content-Disposition: attachment; filename=grades_edX/toy/2012_Fall.csv
+Cache-Control: no-cache, no-store, must-revalidate
+
+"ID","Username","Full Name","edX email","External email","HW 01","HW 02","HW 03","HW 04","HW 05","HW 06","HW 07","HW 08","HW 09","HW 10","HW 11","HW 12","HW Avg","Lab 01","Lab 02","Lab 03","Lab 04","Lab 05","Lab 06","Lab 07","Lab 08","Lab 09","Lab 10","Lab 11","Lab 12","Lab Avg","Midterm","Final"
+"2","u2","Fred Weasley","view2@test.com","","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0.0","0.0"
+'''
+
+        self.assertEqual(respstr, expected_resp, msg)

From b0397ab0b5b5539bce31df8e4513f84c65bef945 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Mon, 3 Sep 2012 12:43:48 -0400
Subject: [PATCH 158/181] fix to be compatible with jenkins test

---
 lms/djangoapps/instructor/tests.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lms/djangoapps/instructor/tests.py b/lms/djangoapps/instructor/tests.py
index 8b8449dbee..4b7b86f3b7 100644
--- a/lms/djangoapps/instructor/tests.py
+++ b/lms/djangoapps/instructor/tests.py
@@ -71,6 +71,7 @@ class TestInstructorDashboardGradeDownloadCSV(ct.PageLoader):
         msg += "instructor dashboard download csv grades: resp = '%s'" % resp
 
         respstr = str(resp).replace('\r','')
+        respstr = respstr.replace('TT_2012','2012')	# jenkins course_id is TT_2012_Fall instead of 2012_Fall?
         #open('idtest.out','w').write(respstr)
 
         expected_resp = '''Vary: Cookie

From 79109a62443932ede354f3b085521238d27bf620 Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Mon, 3 Sep 2012 13:51:20 -0400
Subject: [PATCH 159/181] Apply attr_map translation to metadata loaded from
 policy file as well as from xml.

---
 .../lib/xmodule/xmodule/tests/test_import.py  |  4 ++++
 common/lib/xmodule/xmodule/xml_module.py      | 19 ++++++++++++++++---
 common/test/data/toy/policies/2012_Fall.json  |  3 ++-
 3 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/common/lib/xmodule/xmodule/tests/test_import.py b/common/lib/xmodule/xmodule/tests/test_import.py
index e81d82bf9e..bec8a279c9 100644
--- a/common/lib/xmodule/xmodule/tests/test_import.py
+++ b/common/lib/xmodule/xmodule/tests/test_import.py
@@ -236,6 +236,10 @@ class ImportTestCase(unittest.TestCase):
         # Also check that the grading policy loaded
         self.assertEqual(two_toys.grade_cutoffs['C'], 0.5999)
 
+        # Also check that keys from policy are run through the
+        # appropriate attribute maps -- 'graded' should be True, not 'true'
+        self.assertEqual(toy.metadata['graded'], True)
+
 
     def test_definition_loading(self):
         """When two courses share the same org and course name and
diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py
index 25dc4e0c6e..daeb5dde9b 100644
--- a/common/lib/xmodule/xmodule/xml_module.py
+++ b/common/lib/xmodule/xmodule/xml_module.py
@@ -96,10 +96,13 @@ class XmlDescriptor(XModuleDescriptor):
 
     # A dictionary mapping xml attribute names AttrMaps that describe how
     # to import and export them
+    # Allow json to specify either the string "true", or the bool True.  The string is preferred.
+    to_bool = lambda val: val == 'true' or val == True
+    from_bool = lambda val: str(val).lower()
+    bool_map = AttrMap(to_bool, from_bool)
     xml_attribute_map = {
         # type conversion: want True/False in python, "true"/"false" in xml
-        'graded': AttrMap(lambda val: val == 'true',
-                          lambda val: str(val).lower()),
+        'graded': bool_map,
     }
 
 
@@ -231,6 +234,16 @@ class XmlDescriptor(XModuleDescriptor):
         return metadata
 
 
+    @classmethod
+    def apply_policy(cls, metadata, policy):
+        """
+        Add the keys in policy to metadata, after processing them
+        through the attrmap.  Updates the metadata dict in place.
+        """
+        for attr in policy:
+            attr_map = cls.xml_attribute_map.get(attr, AttrMap())
+            metadata[attr] = attr_map.from_xml(policy[attr])
+
     @classmethod
     def from_xml(cls, xml_data, system, org=None, course=None):
         """
@@ -279,7 +292,7 @@ class XmlDescriptor(XModuleDescriptor):
         # Set/override any metadata specified by policy
         k = policy_key(location)
         if k in system.policy:
-            metadata.update(system.policy[k])
+            cls.apply_policy(metadata, system.policy[k])
 
         return cls(
             system,
diff --git a/common/test/data/toy/policies/2012_Fall.json b/common/test/data/toy/policies/2012_Fall.json
index 5ea437a8e4..23da0a6d96 100644
--- a/common/test/data/toy/policies/2012_Fall.json
+++ b/common/test/data/toy/policies/2012_Fall.json
@@ -2,7 +2,8 @@
     "course/2012_Fall": {
         "graceperiod": "2 days 5 hours 59 minutes 59 seconds",
         "start": "2015-07-17T12:00",
-        "display_name": "Toy Course"
+        "display_name": "Toy Course",
+        "graded": "true"
     },
     "chapter/Overview": {
         "display_name": "Overview"

From f926925ca62f304de93f0f7ac3274b6ecd910c4a Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Mon, 3 Sep 2012 14:12:43 -0400
Subject: [PATCH 160/181] Add a hide_progress_tab course policy flag

- for internal cs50 until we have grade integration set up (only really needed for external version)
---
 common/lib/xmodule/xmodule/course_module.py   | 25 ++++++++++++-------
 common/lib/xmodule/xmodule/xml_module.py      |  1 +
 .../courseware/course_navigation.html         |  4 +--
 3 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/common/lib/xmodule/xmodule/course_module.py b/common/lib/xmodule/xmodule/course_module.py
index 6c8c20014c..681c231c43 100644
--- a/common/lib/xmodule/xmodule/course_module.py
+++ b/common/lib/xmodule/xmodule/course_module.py
@@ -61,7 +61,7 @@ class CourseDescriptor(SequenceDescriptor):
     def __init__(self, system, definition=None, **kwargs):
         super(CourseDescriptor, self).__init__(system, definition, **kwargs)
         self.textbooks = self.definition['data']['textbooks']
-        
+
         self.wiki_slug = self.definition['data']['wiki_slug'] or self.location.course
 
         msg = None
@@ -101,19 +101,19 @@ class CourseDescriptor(SequenceDescriptor):
         for textbook in xml_object.findall("textbook"):
             textbooks.append(cls.Textbook.from_xml_object(textbook))
             xml_object.remove(textbook)
-        
+
         #Load the wiki tag if it exists
         wiki_slug = None
         wiki_tag = xml_object.find("wiki")
         if wiki_tag is not None:
             wiki_slug = wiki_tag.attrib.get("slug", default=None)
             xml_object.remove(wiki_tag)
-        
+
         definition =  super(CourseDescriptor, cls).definition_from_xml(xml_object, system)
-        
+
         definition.setdefault('data', {})['textbooks'] = textbooks
         definition['data']['wiki_slug'] = wiki_slug
-        
+
         return definition
 
     def has_started(self):
@@ -219,7 +219,7 @@ class CourseDescriptor(SequenceDescriptor):
     # there are courses that change the number for different runs. This allows
     # courses to share the same css_class across runs even if they have
     # different numbers.
-    # 
+    #
     # TODO get rid of this as soon as possible or potentially build in a robust
     # way to add in course-specific styling. There needs to be a discussion
     # about the right way to do this, but arjun will address this ASAP. Also
@@ -234,13 +234,20 @@ class CourseDescriptor(SequenceDescriptor):
 
     @property
     def discussion_link(self):
-        """TODO: This is a quick kludge to allow CS50 (and other courses) to 
-        specify their own discussion forums as external links by specifying a 
+        """TODO: This is a quick kludge to allow CS50 (and other courses) to
+        specify their own discussion forums as external links by specifying a
         "discussion_link" in their policy JSON file. This should later get
-        folded in with Syllabus, Course Info, and additional Custom tabs in a 
+        folded in with Syllabus, Course Info, and additional Custom tabs in a
         more sensible framework later."""
         return self.metadata.get('discussion_link', None)
 
+    @property
+    def hide_progress_tab(self):
+        """TODO: same as above, intended to let internal CS50 hide the progress tab
+        until we get grade integration set up."""
+        # Explicit comparison to True because we always want to return a bool.
+        return self.metadata.get('hide_progress_tab') == True
+
     @property
     def title(self):
         return self.display_name
diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py
index daeb5dde9b..070342f453 100644
--- a/common/lib/xmodule/xmodule/xml_module.py
+++ b/common/lib/xmodule/xmodule/xml_module.py
@@ -103,6 +103,7 @@ class XmlDescriptor(XModuleDescriptor):
     xml_attribute_map = {
         # type conversion: want True/False in python, "true"/"false" in xml
         'graded': bool_map,
+        'hide_progress_tab': bool_map,
     }
 
 
diff --git a/lms/templates/courseware/course_navigation.html b/lms/templates/courseware/course_navigation.html
index f7dac9012e..85666bd3d1 100644
--- a/lms/templates/courseware/course_navigation.html
+++ b/lms/templates/courseware/course_navigation.html
@@ -37,7 +37,7 @@ def url_class(url):
       % elif settings.MITX_FEATURES.get('ENABLE_DISCUSSION_SERVICE'):
         <li class="discussion"><a href="${reverse('django_comment_client.forum.views.forum_form_discussion', args=[course.id])}" class="${url_class('discussion')}">Discussion</a></li>
         ## <li class="news"><a href="${reverse('news', args=[course.id])}" class="${url_class('news')}">News</a></li>
-      % endif 
+      % endif
 
       ## This is Askbot, which we should be retiring soon...
       % if settings.MITX_FEATURES.get('ENABLE_DISCUSSION'):
@@ -48,7 +48,7 @@ def url_class(url):
     % if settings.WIKI_ENABLED:
       <li class="wiki"><a href="${reverse('course_wiki', args=[course.id])}" class="${url_class('wiki')}">Wiki</a></li>
     % endif
-    % if user.is_authenticated():
+    % if user.is_authenticated() and not course.hide_progress_tab:
       <li class="profile"><a href="${reverse('progress', args=[course.id])}" class="${url_class('progress')}">Progress</a></li>
     % endif
     % if staff_access:

From 02d58a7ca24318392ebdbda0a62f21836fc048a6 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Mon, 3 Sep 2012 16:49:34 -0400
Subject: [PATCH 161/181] display course load error messages on modulestore
 reload by instructor

---
 lms/djangoapps/instructor/views.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lms/djangoapps/instructor/views.py b/lms/djangoapps/instructor/views.py
index 5a596d3274..96acd66e8e 100644
--- a/lms/djangoapps/instructor/views.py
+++ b/lms/djangoapps/instructor/views.py
@@ -107,6 +107,11 @@ def instructor_dashboard(request, course_id):
             modulestore().try_load_course(data_dir)
             msg += "<br/><p>Course reloaded from %s</p>" % data_dir
             track.views.server_track(request, 'reload %s' % data_dir, {}, page='idashboard')
+            course_errors = modulestore().get_item_errors(course.location)
+            msg += '<ul>'
+            for cmsg, cerr in course_errors:
+                msg += "<li>%s: <pre>%s</pre>" % (cmsg,escape(cerr))
+            msg += '</ul>'
         except Exception as err:
             msg += '<br/><p>Error: %s</p>' % escape(err)
 

From a973c1c21ebdc442b50abb27ceed2753aa574d5d Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Mon, 3 Sep 2012 16:49:55 -0400
Subject: [PATCH 162/181] use module.descriptor.start for start date in
 xmodule_modifiers

---
 common/djangoapps/xmodule_modifiers.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/common/djangoapps/xmodule_modifiers.py b/common/djangoapps/xmodule_modifiers.py
index 6c3c237ea6..9ce6a0235d 100644
--- a/common/djangoapps/xmodule_modifiers.py
+++ b/common/djangoapps/xmodule_modifiers.py
@@ -121,9 +121,9 @@ def add_histogram(get_html, module, user):
         # useful to indicate to staff if problem has been released or not
         now = time.gmtime()
         is_released = "unknown"
-        if hasattr(module,'start'):
-            if module.start is not None:
-                is_released = "<font color='red'>Yes!</font>" if (now > module.start) else "<font color='green'>Not yet</font>"
+        mstart = getattr(module.descriptor,'start')
+        if mstart is not None:
+            is_released = "<font color='red'>Yes!</font>" if (now > mstart) else "<font color='green'>Not yet</font>"
         
         staff_context = {'definition': module.definition.get('data'),
                          'metadata': json.dumps(module.metadata, indent=4),

From dbeca659bd2f607569dcb42e21b0d81094239171 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Mon, 3 Sep 2012 20:32:07 -0400
Subject: [PATCH 163/181] suppress gunicorn error which is being caught by
 errortracker.  The gunicorn error is of this form:

Traceback (most recent call last):
  File "/home/ike/mitx_all/python/local/lib/python2.7/site-packages/gunicorn/workers/sync.py", line 33, in run
    client, addr = self.socket.accept()
  File "/usr/lib/python2.7/socket.py", line 202, in accept
    sock, addr = self._sock.accept()
error: [Errno 11] Resource temporarily unavailable
---
 common/lib/xmodule/xmodule/errortracker.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/common/lib/xmodule/xmodule/errortracker.py b/common/lib/xmodule/xmodule/errortracker.py
index 8ac2903149..e9ea233e09 100644
--- a/common/lib/xmodule/xmodule/errortracker.py
+++ b/common/lib/xmodule/xmodule/errortracker.py
@@ -35,6 +35,11 @@ def make_error_tracker():
         if in_exception_handler():
             exc_str = exc_info_to_str(sys.exc_info())
 
+            # don't display irrelevant gunicorn sync error
+            if (('mitx_all/python/local/lib/python2.7/site-packages/gunicorn/workers/sync.py' in exc_str) and
+                ('[Errno 11] Resource temporarily unavailable' in exc_str)):
+                exc_str = ''
+
         errors.append((msg, exc_str))
 
     return ErrorLog(error_tracker, errors)

From 520d5620d050b57bf2caab38f3fe53fb952f6b21 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Mon, 3 Sep 2012 20:37:33 -0400
Subject: [PATCH 164/181] more general catch for gunicorn sync error

---
 common/lib/xmodule/xmodule/errortracker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/lib/xmodule/xmodule/errortracker.py b/common/lib/xmodule/xmodule/errortracker.py
index e9ea233e09..6accc8b8a7 100644
--- a/common/lib/xmodule/xmodule/errortracker.py
+++ b/common/lib/xmodule/xmodule/errortracker.py
@@ -36,7 +36,7 @@ def make_error_tracker():
             exc_str = exc_info_to_str(sys.exc_info())
 
             # don't display irrelevant gunicorn sync error
-            if (('mitx_all/python/local/lib/python2.7/site-packages/gunicorn/workers/sync.py' in exc_str) and
+            if (('python2.7/site-packages/gunicorn/workers/sync.py' in exc_str) and
                 ('[Errno 11] Resource temporarily unavailable' in exc_str)):
                 exc_str = ''
 

From 93ed042786a0dd9a47be2932d77dd3b47d5dbd21 Mon Sep 17 00:00:00 2001
From: Arjun Singh <arjun810@gmail.com>
Date: Mon, 3 Sep 2012 19:37:48 -0700
Subject: [PATCH 165/181] Make show answer work for choicegroups +
 javascriptinputs

---
 .../lib/capa/capa/templates/choicegroup.html  |  2 +-
 .../capa/capa/templates/javascriptinput.html  |  2 +-
 .../lib/xmodule/xmodule/css/capa/display.scss |  6 +++
 .../xmodule/js/src/capa/display.coffee        | 47 ++++++++++++++++++-
 4 files changed, 54 insertions(+), 3 deletions(-)

diff --git a/common/lib/capa/capa/templates/choicegroup.html b/common/lib/capa/capa/templates/choicegroup.html
index 2bed2d875e..cb11350a63 100644
--- a/common/lib/capa/capa/templates/choicegroup.html
+++ b/common/lib/capa/capa/templates/choicegroup.html
@@ -1,4 +1,4 @@
-<form class="choicegroup">
+<form class="choicegroup capa_inputtype" id="inputtype_${id}">
   
   % for choice_id, choice_description in choices:
   <label for="input_${id}_${choice_id}">  <input type="${input_type}" name="input_${id}${name_array_suffix}" id="input_${id}_${choice_id}" value="${choice_id}"
diff --git a/common/lib/capa/capa/templates/javascriptinput.html b/common/lib/capa/capa/templates/javascriptinput.html
index bafcb23cc7..8b4c8f7115 100644
--- a/common/lib/capa/capa/templates/javascriptinput.html
+++ b/common/lib/capa/capa/templates/javascriptinput.html
@@ -1,4 +1,4 @@
-<form class="javascriptinput capa_inputtype">
+<form class="javascriptinput capa_inputtype" id="inputtype_${id}">
   <input type="hidden" name="input_${id}" id="input_${id}" class="javascriptinput_input"/>
     <div class="javascriptinput_data" data-display_class="${display_class}" 
         data-problem_state="${problem_state}" data-params="${params}" 
diff --git a/common/lib/xmodule/xmodule/css/capa/display.scss b/common/lib/xmodule/xmodule/css/capa/display.scss
index fd871b0b03..7a9e7532e1 100644
--- a/common/lib/xmodule/xmodule/css/capa/display.scss
+++ b/common/lib/xmodule/xmodule/css/capa/display.scss
@@ -32,6 +32,12 @@ section.problem {
       display: inline;
   }
 
+  .choicegroup {
+    label.choicegroup_correct:after {
+      content: url('../images/correct-icon.png');
+    }
+  }
+
 
   div {
     p {
diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee
index 098b79d9cf..e27b46d04e 100644
--- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee
+++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee
@@ -84,11 +84,14 @@ class @Problem
   # stuff if a div w a class is found
 
   setupInputTypes: =>
+    @inputtypeDisplays = {}
     @el.find(".capa_inputtype").each (index, inputtype) =>
       classes = $(inputtype).attr('class').split(' ')
+      id = $(inputtype).attr('id')
       for cls in classes
         setupMethod = @inputtypeSetupMethods[cls]
-        setupMethod(inputtype) if setupMethod?
+        if setupMethod?
+          @inputtypeDisplays[id] = setupMethod(inputtype)
 
   executeProblemScripts: (callback=null) ->
 
@@ -248,6 +251,17 @@ class @Problem
               @$("label[for='input_#{key}_#{choice}']").attr correct_answer: 'true'
           else
             @$("#answer_#{key}, #solution_#{key}").html(value)
+
+        # TODO remove the above once everything is extracted into its own
+        # inputtype functions.
+
+        @el.find(".capa_inputtype").each (index, inputtype) =>
+            classes = $(inputtype).attr('class').split(' ')
+            for cls in classes
+              display = @inputtypeDisplays[$(inputtype).attr('id')]
+              showMethod = @inputtypeShowAnswerMethods[cls]
+              showMethod(inputtype, display, answers) if showMethod?
+
         MathJax.Hub.Queue ["Typeset", MathJax.Hub]
         @$('.show').val 'Hide Answer'
         @el.addClass 'showed'
@@ -258,6 +272,13 @@ class @Problem
       @el.removeClass 'showed'
       @$('.show').val 'Show Answer'
 
+      @el.find(".capa_inputtype").each (index, inputtype) =>
+        display = @inputtypeDisplays[$(inputtype).attr('id')]
+        classes = $(inputtype).attr('class').split(' ')
+        for cls in classes
+          hideMethod = @inputtypeHideAnswerMethods[cls]
+          hideMethod(inputtype, display) if hideMethod?
+
   gentle_alert: (msg) =>
     if @el.find('.capa_alert').length
       @el.find('.capa_alert').remove()
@@ -314,3 +335,27 @@ class @Problem
 
       display = new displayClass(problemState, submission, evaluation, container, submissionField, params)
       display.render()
+
+      return display
+
+  inputtypeShowAnswerMethods:
+    choicegroup: (element, display, answers) =>
+      element = $(element)
+      for key, value of answers
+        element.find('input').attr('disabled', 'disabled')
+        for choice in value
+          element.find("label[for='input_#{key}_#{choice}']").addClass 'choicegroup_correct'
+
+    javascriptinput: (element, display, answers) =>
+      answer_id = $(element).attr('id').split("_")[1...].join("_")
+      answer = JSON.parse(answers[answer_id])
+      display.showAnswer(answer)
+
+  inputtypeHideAnswerMethods:
+    choicegroup: (element, display) =>
+      element = $(element)
+      element.find('input').attr('disabled', null)
+      element.find('label').removeClass('choicegroup_correct')
+
+    javascriptinput: (element, display) =>
+      display.hideAnswer()

From 258274f2a7d265e2af88079fb87d2346b8444128 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Tue, 4 Sep 2012 10:25:28 -0400
Subject: [PATCH 166/181] add string to model class unicode

---
 common/djangoapps/student/models.py | 2 +-
 lms/templates/courses.html          | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py
index 529a6f1298..c37b9fce16 100644
--- a/common/djangoapps/student/models.py
+++ b/common/djangoapps/student/models.py
@@ -187,7 +187,7 @@ class CourseEnrollment(models.Model):
         unique_together = (('user', 'course_id'), )
 
     def __unicode__(self):
-        return "%s: %s (%s)" % (self.user, self.course_id, self.created)
+        return "[CourseEnrollment] %s: %s (%s)" % (self.user, self.course_id, self.created)
 
 
 @receiver(post_save, sender=CourseEnrollment)
diff --git a/lms/templates/courses.html b/lms/templates/courses.html
index 25930184a9..414292012c 100644
--- a/lms/templates/courses.html
+++ b/lms/templates/courses.html
@@ -19,6 +19,12 @@
   <section class="container">
     ## I'm removing this for now since we aren't using it for the fall.
     ## <%include file="course_filter.html" />
+
+<%!
+from django.core.urlresolvers import reverse
+%>    
+   <h1>For 8.01 Reading Questions, click on     <a href="${reverse('courses')}">Find Courses</a></h1>
+
     <section class="courses">
       <section class='university-column'>
         %for course in universities['MITx']:

From 4f8161df84b22839f1435d333136480331545256 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Tue, 4 Sep 2012 11:10:22 -0400
Subject: [PATCH 167/181] address two comments in
 https://github.com/MITx/mitx/pull/615

---
 common/djangoapps/xmodule_modifiers.py | 1 +
 doc/development.md                     | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/common/djangoapps/xmodule_modifiers.py b/common/djangoapps/xmodule_modifiers.py
index 9ce6a0235d..066d83ed3e 100644
--- a/common/djangoapps/xmodule_modifiers.py
+++ b/common/djangoapps/xmodule_modifiers.py
@@ -119,6 +119,7 @@ def add_histogram(get_html, module, user):
         source_file = module.metadata.get('source_file','')	# source used to generate the problem XML, eg latex or word
 
         # useful to indicate to staff if problem has been released or not
+        # TODO (ichuang): use _has_access_descriptor.can_load in lms.courseware.access, instead of now>mstart comparison here
         now = time.gmtime()
         is_released = "unknown"
         mstart = getattr(module.descriptor,'start')
diff --git a/doc/development.md b/doc/development.md
index fcf0b7a40f..7770ef1a11 100644
--- a/doc/development.md
+++ b/doc/development.md
@@ -94,3 +94,5 @@ course content can be setup to trigger an automatic reload when changes are push
 
     The mitx server will then do "git reset --hard HEAD; git clean -f -d; git pull origin" in that directory.  After the pull,
     it will reload the modulestore for that course.
+
+Note that the gitreload-based workflow is not meant for deployments on AWS (or elsewhere) which use collectstatic, since collectstatic is not run by a gitreload event.

From d6430570527d9d2ae9af85b32259d2ffab43e8a9 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Tue, 4 Sep 2012 11:21:18 -0400
Subject: [PATCH 168/181] count all types of modules, not just childless ones

---
 lms/djangoapps/instructor/views.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/lms/djangoapps/instructor/views.py b/lms/djangoapps/instructor/views.py
index 96acd66e8e..92b2401216 100644
--- a/lms/djangoapps/instructor/views.py
+++ b/lms/djangoapps/instructor/views.py
@@ -345,10 +345,8 @@ def compute_course_stats(course):
 
     def walk(module):
         children = module.get_children()
-        if not children:
-            category = module.__class__.__name__ 	# HtmlDescriptor, CapaDescriptor, ...
-            counts[category] += 1
-            return
+        category = module.__class__.__name__ 	# HtmlDescriptor, CapaDescriptor, ...
+        counts[category] += 1
         for c in children:
             walk(c)
 

From c23b31c6ee75178118de23d944d9a035e205e0eb Mon Sep 17 00:00:00 2001
From: Victor Shnayder <victor@mitx.mit.edu>
Date: Tue, 4 Sep 2012 16:20:39 -0400
Subject: [PATCH 169/181] Remove lots of loud warnings about duplicate url
 names for things which don't store state, and have the same hash.

- should make 6.002 discussion tags happier
---
 common/lib/xmodule/xmodule/modulestore/xml.py | 38 ++++++++++++-------
 1 file changed, 24 insertions(+), 14 deletions(-)

diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py
index 23a5473292..d46fc04410 100644
--- a/common/lib/xmodule/xmodule/modulestore/xml.py
+++ b/common/lib/xmodule/xmodule/modulestore/xml.py
@@ -62,6 +62,9 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                 """
                 # VS[compat]. Take this out once course conversion is done (perhaps leave the uniqueness check)
 
+                # tags that really need unique names--they store (or should store) state.
+                need_uniq_names = ('problem', 'sequence', 'video', 'course', 'chapter')
+
                 attr = xml_data.attrib
                 tag = xml_data.tag
                 id = lambda x: x
@@ -86,12 +89,15 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                     orig_name = "_" + orig_name if orig_name is not None else ""
                     return tag + orig_name + "_" + hashlib.sha1(xml).hexdigest()[:12]
 
+                def looks_like_fallback(tag, url_name):
+                    """Does this look like something that came from fallback_name()?"""
+                    return url_name.startswith(tag) and re.search('[0-9a-fA-F]{12}$', url_name)
+
                 # Fallback if there was nothing we could use:
                 if url_name is None or url_name == "":
                     url_name = fallback_name()
                     # Don't log a warning--we don't need this in the log.  Do
                     # put it in the error tracker--content folks need to see it.
-                    need_uniq_names = ('problem', 'sequence', 'video', 'course', 'chapter')
 
                     if tag in need_uniq_names:
                         error_tracker("PROBLEM: no name of any kind specified for {tag}.  Student "
@@ -105,20 +111,24 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
 
                 # Make sure everything is unique
                 if url_name in self.used_names[tag]:
-                    msg = ("Non-unique url_name in xml.  This may break state tracking for content."
-                           "  url_name={0}.  Content={1}".format(url_name, xml[:100]))
-                    error_tracker("PROBLEM: " + msg)
-                    log.warning(msg)
-                    # Just set name to fallback_name--if there are multiple things with the same fallback name,
-                    # they are actually identical, so it's fragile, but not immediately broken.
+                    # Always complain about modules that store state.  If it
+                    # doesn't store state, don't complain about things that are
+                    # hashed.
+                    if tag in need_uniq_names or not looks_like_fallback(tag, url_name):
+                        msg = ("Non-unique url_name in xml.  This may break state tracking for content."
+                               "  url_name={0}.  Content={1}".format(url_name, xml[:100]))
+                        error_tracker("PROBLEM: " + msg)
+                        log.warning(msg)
+                        # Just set name to fallback_name--if there are multiple things with the same fallback name,
+                        # they are actually identical, so it's fragile, but not immediately broken.
 
-                    # TODO (vshnayder): if the tag is a pointer tag, this will
-                    # break the content because we won't have the right link.
-                    # That's also a legitimate attempt to reuse the same content
-                    # from multiple places.  Once we actually allow that, we'll
-                    # need to update this to complain about non-unique names for
-                    # definitions, but allow multiple uses.
-                    url_name = fallback_name(url_name)
+                        # TODO (vshnayder): if the tag is a pointer tag, this will
+                        # break the content because we won't have the right link.
+                        # That's also a legitimate attempt to reuse the same content
+                        # from multiple places.  Once we actually allow that, we'll
+                        # need to update this to complain about non-unique names for
+                        # definitions, but allow multiple uses.
+                        url_name = fallback_name(url_name)
 
                 self.used_names[tag].add(url_name)
                 xml_data.set('url_name', url_name)

From 4cbdf90d5e2f77ea209616d7fb9ed51b603f07e2 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Tue, 4 Sep 2012 18:26:22 -0400
Subject: [PATCH 170/181] cleaned up access.py - now has
 _has_instructor_access_to_loation

---
 lms/djangoapps/courseware/access.py | 62 +++++++++++++++++++++--------
 1 file changed, 45 insertions(+), 17 deletions(-)

diff --git a/lms/djangoapps/courseware/access.py b/lms/djangoapps/courseware/access.py
index 4f35aa98fa..2612efedb9 100644
--- a/lms/djangoapps/courseware/access.py
+++ b/lms/djangoapps/courseware/access.py
@@ -153,7 +153,7 @@ def _has_access_course_desc(user, course, action):
         'enroll': can_enroll,
         'see_exists': see_exists,
         'staff': lambda: _has_staff_access_to_descriptor(user, course),
-        'instructor': lambda: _has_staff_access_to_descriptor(user, course, require_instructor=True),
+        'instructor': lambda: _has_instructor_access_to_descriptor(user, course),
         }
 
     return _dispatch(checkers, action, user, course)
@@ -314,6 +314,7 @@ def _course_staff_group_name(location):
     """
     return 'staff_%s' % Location(location).course
 
+
 def _course_instructor_group_name(location):
     """
     Get the name of the instructor group for a location.  Right now, that's instructor_COURSE.
@@ -323,6 +324,7 @@ def _course_instructor_group_name(location):
     """
     return 'instructor_%s' % Location(location).course
 
+
 def _has_global_staff_access(user):
     if user.is_staff:
         debug("Allow: user.is_staff")
@@ -332,19 +334,28 @@ def _has_global_staff_access(user):
         return False
 
 
-def _has_staff_access_to_location(user, location, require_instructor=False):
+def _has_instructor_access_to_location(user, location):
+    return _has_access_to_location(user, location, 'instructor')
+
+
+def _has_staff_access_to_location(user, location):
+    return _has_access_to_location(user, location, 'staff')
+
+
+def _has_access_to_location(user, location, access_level):
     '''
-    Returns True if the given user has staff access to a location.  For now this
-    is equivalent to having staff access to the course location.course.
+    Returns True if the given user has access_level (= staff or
+    instructor) access to a location.  For now this is equivalent to
+    having staff / instructor access to the course location.course.
 
-    If require_instructor=True, then user must be in instructor_* group.
-
-    This means that user is in the staff_* group, or is an overall admin.
+    This means that user is in the staff_* group or instructor_* group, or is an overall admin.
 
     TODO (vshnayder): this needs to be changed to allow per-course_id permissions, not per-course
     (e.g. staff in 2012 is different from 2013, but maybe some people always have access)
 
     course is a string: the course field of the location being accessed.
+    location = location
+    access_level = string, either "staff" or "instructor"
     '''
     if user is None or (not user.is_authenticated()):
         debug("Deny: no user or anon user")
@@ -355,29 +366,46 @@ def _has_staff_access_to_location(user, location, require_instructor=False):
 
     # If not global staff, is the user in the Auth group for this class?
     user_groups = [g.name for g in user.groups.all()]
-    staff_group = _course_staff_group_name(location)
-    if not require_instructor:
+
+    if access_level == 'staff':
+        staff_group = _course_staff_group_name(location)
         if staff_group in user_groups:
             debug("Allow: user in group %s", staff_group)
             return True
-    instructor_group = _course_instructor_group_name(location)
-    if instructor_group in user_groups:
-        debug("Allow: user in group %s", instructor_group)
-        return True
-    debug("Deny: user not in group %s", staff_group)
+        debug("Deny: user not in group %s", staff_group)
+
+    if access_level == 'instructor' or access_level == 'staff': 	# instructors get staff privileges
+        instructor_group = _course_instructor_group_name(location)
+        if instructor_group in user_groups:
+            debug("Allow: user in group %s", instructor_group)
+            return True
+        debug("Deny: user not in group %s", instructor_group)
+
+    else:
+        log.debug("Error in access._has_access_to_location access_level=%s unknown" % access_level)
+
     return False
 
+
 def _has_staff_access_to_course_id(user, course_id):
     """Helper method that takes a course_id instead of a course name"""
     loc = CourseDescriptor.id_to_location(course_id)
     return _has_staff_access_to_location(user, loc)
 
 
-def _has_staff_access_to_descriptor(user, descriptor, require_instructor=False):
+def _has_instructor_access_to_descriptor(user, descriptor):
     """Helper method that checks whether the user has staff access to
     the course of the location.
 
-    location: something that can be passed to Location
+    descriptor: something that has a location attribute
     """
-    return _has_staff_access_to_location(user, descriptor.location, require_instructor=require_instructor)
+    return _has_instructor_access_to_location(user, descriptor.location)
+
+def _has_staff_access_to_descriptor(user, descriptor):
+    """Helper method that checks whether the user has staff access to
+    the course of the location.
+
+    descriptor: something that has a location attribute
+    """
+    return _has_staff_access_to_location(user, descriptor.location)
 

From 0c3d0542532540e3eb29482d4f85d759b7b762be Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Tue, 4 Sep 2012 18:36:26 -0400
Subject: [PATCH 171/181] fix: university domain should not be determined by
 HTTP_HOST for dev or behind proxy

---
 common/djangoapps/student/views.py | 5 ++++-
 lms/envs/common.py                 | 3 +++
 lms/envs/dev.py                    | 1 +
 lms/envs/dev_ike.py                | 4 ++++
 4 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py
index 27adf485f0..486aba12f6 100644
--- a/common/djangoapps/student/views.py
+++ b/common/djangoapps/student/views.py
@@ -75,8 +75,11 @@ def index(request, extra_context={}, user=None):
         entry.summary = soup.getText()
 
     # The course selection work is done in courseware.courses.
+    domain = settings.MITX_FEATURES.get('FORCE_UNIVERSITY_DOMAIN')	# normally False
+    if not domain:
+        domain = request.META.get('HTTP_HOST')
     universities = get_courses_by_university(None,
-                                             domain=request.META.get('HTTP_HOST'))
+                                             domain=domain)
     context = {'universities': universities, 'entries': entries}
     context.update(extra_context)
     return render_to_response('index.html', context)
diff --git a/lms/envs/common.py b/lms/envs/common.py
index cf9a767d9f..7e868f0a34 100644
--- a/lms/envs/common.py
+++ b/lms/envs/common.py
@@ -64,6 +64,9 @@ MITX_FEATURES = {
     # university to use for branding purposes
     'SUBDOMAIN_BRANDING': False,
 
+    'FORCE_UNIVERSITY_DOMAIN': False,	# set this to the university domain to use, as an override to HTTP_HOST
+                                        # set to None to do no university selection
+
     'ENABLE_TEXTBOOK' : True,
     'ENABLE_DISCUSSION' : False,
     'ENABLE_DISCUSSION_SERVICE': True,
diff --git a/lms/envs/dev.py b/lms/envs/dev.py
index 974b8c9fd6..0427938b70 100644
--- a/lms/envs/dev.py
+++ b/lms/envs/dev.py
@@ -17,6 +17,7 @@ MITX_FEATURES['DISABLE_START_DATES'] = True
 MITX_FEATURES['ENABLE_SQL_TRACKING_LOGS'] = True
 MITX_FEATURES['SUBDOMAIN_COURSE_LISTINGS'] = False  # Enable to test subdomains--otherwise, want all courses to show up
 MITX_FEATURES['SUBDOMAIN_BRANDING'] = True
+MITX_FEATURES['FORCE_UNIVERSITY_DOMAIN'] = None		# show all university courses if in dev (ie don't use HTTP_HOST)
 
 WIKI_ENABLED = True
 
diff --git a/lms/envs/dev_ike.py b/lms/envs/dev_ike.py
index 297b179fae..0be9146fd4 100644
--- a/lms/envs/dev_ike.py
+++ b/lms/envs/dev_ike.py
@@ -18,6 +18,7 @@ MITX_FEATURES['ENABLE_DISCUSSION'] = False
 MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True	  # require that user be in the staff_* group to be able to enroll
 MITX_FEATURES['SUBDOMAIN_COURSE_LISTINGS'] = False
 MITX_FEATURES['SUBDOMAIN_BRANDING'] = False
+MITX_FEATURES['FORCE_UNIVERSITY_DOMAIN'] = None		# show all university courses if in dev (ie don't use HTTP_HOST)
 
 MITX_FEATURES['DISABLE_START_DATES'] = True
 # MITX_FEATURES['USE_DJANGO_PIPELINE']=False      # don't recompile scss
@@ -28,6 +29,9 @@ if ('edxvm' in myhost) or ('ocw' in myhost):
     MITX_FEATURES['USE_XQA_SERVER'] = 'https://qisx.mit.edu/xqa'	# needs to be ssl or browser blocks it
     MITX_FEATURES['USE_DJANGO_PIPELINE']=False      # don't recompile scss
 
+if ('ocw' in myhost):
+    MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = False
+
 if ('domU' in myhost):
     EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
     MITX_FEATURES['REROUTE_ACTIVATION_EMAIL'] = 'ichuang@mitx.mit.edu'	# nonempty string = address for all activation emails

From 98eb7645259de80f4247ea4b6a46b428d906742b Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Tue, 4 Sep 2012 18:56:49 -0400
Subject: [PATCH 172/181] cleaned up instructor dashboard CSV unit test code,
 per https://github.com/MITx/mitx/pull/615

---
 lms/djangoapps/instructor/tests.py | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/lms/djangoapps/instructor/tests.py b/lms/djangoapps/instructor/tests.py
index 4b7b86f3b7..e948771d6d 100644
--- a/lms/djangoapps/instructor/tests.py
+++ b/lms/djangoapps/instructor/tests.py
@@ -65,22 +65,20 @@ class TestInstructorDashboardGradeDownloadCSV(ct.PageLoader):
         course = self.toy
         url = reverse('instructor_dashboard', kwargs={'course_id': course.id})
         msg = "url = %s\n" % url
-        resp = self.client.post(url, {
-            'action': 'Download CSV of all student grades for this course',
-            })
-        msg += "instructor dashboard download csv grades: resp = '%s'" % resp
+        response = self.client.post(url, {'action': 'Download CSV of all student grades for this course',
+                                          })
+        msg += "instructor dashboard download csv grades: response = '%s'\n" % response
 
-        respstr = str(resp).replace('\r','')
-        respstr = respstr.replace('TT_2012','2012')	# jenkins course_id is TT_2012_Fall instead of 2012_Fall?
-        #open('idtest.out','w').write(respstr)
+        self.assertEqual(response['Content-Type'],'text/csv',msg)
 
-        expected_resp = '''Vary: Cookie
-Content-Type: text/csv
-Content-Disposition: attachment; filename=grades_edX/toy/2012_Fall.csv
-Cache-Control: no-cache, no-store, must-revalidate
+        cdisp = response['Content-Disposition'].replace('TT_2012','2012')  # jenkins course_id is TT_2012_Fall instead of 2012_Fall?
+        msg += "cdisp = '%s'\n" % cdisp
+        self.assertEqual(cdisp,'attachment; filename=grades_edX/toy/2012_Fall.csv',msg)
 
-"ID","Username","Full Name","edX email","External email","HW 01","HW 02","HW 03","HW 04","HW 05","HW 06","HW 07","HW 08","HW 09","HW 10","HW 11","HW 12","HW Avg","Lab 01","Lab 02","Lab 03","Lab 04","Lab 05","Lab 06","Lab 07","Lab 08","Lab 09","Lab 10","Lab 11","Lab 12","Lab Avg","Midterm","Final"
+        body = response.content.replace('\r','')
+        msg += "body = '%s'\n" % body
+
+        expected_body = '''"ID","Username","Full Name","edX email","External email","HW 01","HW 02","HW 03","HW 04","HW 05","HW 06","HW 07","HW 08","HW 09","HW 10","HW 11","HW 12","HW Avg","Lab 01","Lab 02","Lab 03","Lab 04","Lab 05","Lab 06","Lab 07","Lab 08","Lab 09","Lab 10","Lab 11","Lab 12","Lab Avg","Midterm","Final"
 "2","u2","Fred Weasley","view2@test.com","","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0.0","0.0"
 '''
-
-        self.assertEqual(respstr, expected_resp, msg)
+        self.assertEqual(body, expected_body, msg)

From 5c90ec19d9552c132979c3b04b6e00fe49fe3270 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Tue, 4 Sep 2012 19:08:28 -0400
Subject: [PATCH 173/181] clarify that /gitreload is behind
 ENABLE_LMS_MIGRATION in docs

---
 doc/development.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/development.md b/doc/development.md
index 7770ef1a11..b4ac52d202 100644
--- a/doc/development.md
+++ b/doc/development.md
@@ -96,3 +96,6 @@ course content can be setup to trigger an automatic reload when changes are push
     it will reload the modulestore for that course.
 
 Note that the gitreload-based workflow is not meant for deployments on AWS (or elsewhere) which use collectstatic, since collectstatic is not run by a gitreload event.
+
+Also, the gitreload feature needs MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True in the django settings.
+

From 566b9694c4f3b5de25906314c79bd0c3f9f56e0c Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Tue, 4 Sep 2012 19:47:37 -0400
Subject: [PATCH 174/181] revert inadvertent change to course.html

---
 lms/templates/courses.html | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/lms/templates/courses.html b/lms/templates/courses.html
index 414292012c..25930184a9 100644
--- a/lms/templates/courses.html
+++ b/lms/templates/courses.html
@@ -19,12 +19,6 @@
   <section class="container">
     ## I'm removing this for now since we aren't using it for the fall.
     ## <%include file="course_filter.html" />
-
-<%!
-from django.core.urlresolvers import reverse
-%>    
-   <h1>For 8.01 Reading Questions, click on     <a href="${reverse('courses')}">Find Courses</a></h1>
-
     <section class="courses">
       <section class='university-column'>
         %for course in universities['MITx']:

From 0f1db7969d7beed1e6eb6d3ea875788a0b5032aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Andr=C3=A9s=20Rocha?= <rocha@edx.org>
Date: Tue, 4 Sep 2012 20:53:51 -0400
Subject: [PATCH 175/181] [#35491529] Small corrections to OpenID provider
 login page

---
 lms/templates/provider_login.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lms/templates/provider_login.html b/lms/templates/provider_login.html
index 620e0c4191..a98b2ab32b 100644
--- a/lms/templates/provider_login.html
+++ b/lms/templates/provider_login.html
@@ -40,11 +40,11 @@
       <div id="login_error" class="modal-form-error" style="display: block;">Email or password is incorrect.</div>
 %endif
       <label>E-mail</label>
-      <input type="text" name="email" placeholder="E-mail" tabindex="1" />
+      <input type="text" name="email" placeholder="E-mail" tabindex="1" autofocus="autofocus" />
       <label>Password</label>
       <input type="password" name="password" placeholder="Password" tabindex="2" />
       <div class="submit">
-        <input name="submit" type="submit" value="Access My Courses and Return To ${return_to}" tabindex="3" />
+        <input name="submit" type="submit" value="Return To ${return_to}" tabindex="3" />
       </div>
     </form>
   </div>

From 319e6260d0314d484aa212f2537d92c714e3dda5 Mon Sep 17 00:00:00 2001
From: David Ormsbee <dave@edx.org>
Date: Tue, 4 Sep 2012 21:11:34 -0400
Subject: [PATCH 176/181] Upped the number of threads per discussion page to 8,
 per Tony's request.

---
 lms/djangoapps/django_comment_client/forum/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lms/djangoapps/django_comment_client/forum/views.py b/lms/djangoapps/django_comment_client/forum/views.py
index 58b6d30182..c24c8b79bd 100644
--- a/lms/djangoapps/django_comment_client/forum/views.py
+++ b/lms/djangoapps/django_comment_client/forum/views.py
@@ -20,7 +20,7 @@ import django_comment_client.utils as utils
 import comment_client as cc
 
 
-THREADS_PER_PAGE = 5
+THREADS_PER_PAGE = 8
 PAGES_NEARBY_DELTA = 2
 
 

From d56d235fc9284a208b986b509ae640ae8497aacc Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Tue, 4 Sep 2012 23:02:51 -0400
Subject: [PATCH 177/181] minor change to create_user command, don't fail on
 kerberos lookup failure

---
 .../lms_migration/management/commands/create_user.py         | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lms/djangoapps/lms_migration/management/commands/create_user.py b/lms/djangoapps/lms_migration/management/commands/create_user.py
index 333608d467..7d39accc44 100644
--- a/lms/djangoapps/lms_migration/management/commands/create_user.py
+++ b/lms/djangoapps/lms_migration/management/commands/create_user.py
@@ -67,7 +67,10 @@ class Command(BaseCommand):
             password = GenPasswd(12)
             
             # get name from kerberos
-            kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip()
+            try:
+                kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip()
+            except:
+                kname = ''
             name = raw_input('Full name: [%s] ' % kname).strip()
             if name=='':
                 name = kname

From 44722e86e177995e0e2cb89aba599f581df6ea88 Mon Sep 17 00:00:00 2001
From: David Ormsbee <dave@edx.org>
Date: Wed, 5 Sep 2012 08:20:06 -0400
Subject: [PATCH 178/181] Quick fix for prod issue -- making view courseware
 link from course about page always work.

---
 lms/templates/portal/course_about.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lms/templates/portal/course_about.html b/lms/templates/portal/course_about.html
index d6c731431e..449878680f 100644
--- a/lms/templates/portal/course_about.html
+++ b/lms/templates/portal/course_about.html
@@ -71,14 +71,14 @@
                       course_target = reverse('about_course', args=[course.id])
                   show_link = settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION')
                 %>
-	        %if show_link:
+	        %if show_link or show_courseware_link:
                 <a href="${course_target}">
                 %endif
                 <span class="register disabled">You are registered for this course (${course.number})</span>
                 % if show_courseware_link:
                   <strong>View Courseware</strong>
                 % endif
-	        %if show_link:
+	        %if show_link or show_courseware_link:
                 </a>
                 %endif
             %else:

From 2d42c91b9d99b5d10bed2fa309aa85da97466024 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20Andr=C3=A9s=20Rocha?= <rocha@edx.org>
Date: Wed, 5 Sep 2012 10:22:36 -0400
Subject: [PATCH 179/181] Cleanup of 'About Course' page link to its Courseware

---
 lms/djangoapps/courseware/views.py     | 13 +++++++++--
 lms/templates/portal/course_about.html | 30 +++++++++-----------------
 2 files changed, 21 insertions(+), 22 deletions(-)

diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py
index 71ec687cf6..0fd799e756 100644
--- a/lms/djangoapps/courseware/views.py
+++ b/lms/djangoapps/courseware/views.py
@@ -264,10 +264,19 @@ def registered_for_course(course, user):
 def course_about(request, course_id):
     course = get_course_with_access(request.user, course_id, 'see_exists')
     registered = registered_for_course(course, request.user)
-    show_courseware_link = has_access(request.user, course, 'load')
+
+    if has_access(request.user, course, 'load'):
+        course_target = reverse('info', args=[course.id])
+    else:
+        course_target = reverse('about_course', args=[course.id])
+
+    show_courseware_link = (has_access(request.user, course, 'load') or
+                            settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'))
+
     return render_to_response('portal/course_about.html',
                               {'course': course,
-                               'registered': registered, 
+                               'registered': registered,
+                               'course_target': course_target,
                                'show_courseware_link' : show_courseware_link})
 
 
diff --git a/lms/templates/portal/course_about.html b/lms/templates/portal/course_about.html
index 449878680f..5fabab7db5 100644
--- a/lms/templates/portal/course_about.html
+++ b/lms/templates/portal/course_about.html
@@ -61,33 +61,23 @@
         </hgroup>
 
         <div class="main-cta">
-          %if user.is_authenticated():
+        %if user.is_authenticated():
             %if registered:
-   	        <%
-		  ## TODO: move this logic into a view
-                  if has_access(user, course, 'load'):
-                      course_target = reverse('info', args=[course.id])
-                  else:
-                      course_target = reverse('about_course', args=[course.id])
-                  show_link = settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION')
-                %>
-	        %if show_link or show_courseware_link:
-                <a href="${course_target}">
+	              %if show_courseware_link:
+                    <a href="${course_target}">
                 %endif
                 <span class="register disabled">You are registered for this course (${course.number})</span>
-                % if show_courseware_link:
-                  <strong>View Courseware</strong>
-                % endif
-	        %if show_link or show_courseware_link:
-                </a>
+                %if show_courseware_link:
+                    <strong>View Courseware</strong>
+                    </a>
                 %endif
             %else:
-                <a href="#" class="register">Register for ${course.number}</a>
-	        <div id="register_message"></div>
+               <a href="#" class="register">Register for ${course.number}</a>
+	             <div id="register_message"></div>
             %endif
-          %else:
+        %else:
             <a href="#signup-modal" class="register" rel="leanModal" data-notice='You must Sign Up or <a href="#login-modal" rel="leanModal">Log In</a> to enroll.'>Register for ${course.number}</a>
-          %endif
+        %endif
         </div>
 
       </section>

From 34ea1e0b95b7b7722f58e89055119519d1b8fc9d Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Wed, 5 Sep 2012 11:19:02 -0400
Subject: [PATCH 180/181] fix FORCE_UNIVERSITY_DOMAIN check in student.views

---
 common/djangoapps/student/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py
index 486aba12f6..cbb12e44cc 100644
--- a/common/djangoapps/student/views.py
+++ b/common/djangoapps/student/views.py
@@ -76,7 +76,7 @@ def index(request, extra_context={}, user=None):
 
     # The course selection work is done in courseware.courses.
     domain = settings.MITX_FEATURES.get('FORCE_UNIVERSITY_DOMAIN')	# normally False
-    if not domain:
+    if domain==False:				# do explicit check, because domain=None is valid
         domain = request.META.get('HTTP_HOST')
     universities = get_courses_by_university(None,
                                              domain=domain)

From 77029076625ee96565a8cd94d59466b2794b394d Mon Sep 17 00:00:00 2001
From: Calen Pennington <calen.pennington@gmail.com>
Date: Wed, 5 Sep 2012 13:29:30 -0400
Subject: [PATCH 181/181] Make module system handle not having a user, and pass
 a user in when previewing in the cms

---
 cms/djangoapps/contentstore/views.py   | 1 +
 common/lib/xmodule/xmodule/x_module.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py
index 6d653db6cc..490f49a41c 100644
--- a/cms/djangoapps/contentstore/views.py
+++ b/cms/djangoapps/contentstore/views.py
@@ -215,6 +215,7 @@ def preview_module_system(request, preview_id, descriptor):
         render_template=render_from_lms,
         debug=True,
         replace_urls=replace_urls,
+        user=request.user,
     )
 
 
diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py
index 9d65ce65f2..be61498de6 100644
--- a/common/lib/xmodule/xmodule/x_module.py
+++ b/common/lib/xmodule/xmodule/x_module.py
@@ -771,7 +771,7 @@ class ModuleSystem(object):
         self.replace_urls = replace_urls
         self.node_path = node_path
         self.anonymous_student_id = anonymous_student_id
-        self.user_is_staff = user.is_staff
+        self.user_is_staff = user is not None and user.is_staff
 
     def get(self, attr):
         '''	provide uniform access to attributes (like etree).'''