Merge remote-tracking branch 'origin/master' into MITx/feature/bridger/fast_course_grading

Conflicts: lms/djangoapps/courseware/module_render.py lms/djangoapps/courseware/views.py
2012-08-06 16:49:05 -04:00
parent baa2b14405 24f85bf202
commit 4fedca5bf9
32 changed files with 847 additions and 234 deletions
--- a/common/djangoapps/external_auth/views.py
+++ b/common/djangoapps/external_auth/views.py
@@ -150,7 +150,7 @@ def edXauth_signup(request, eamap=None):
    context = {'has_extauth_info': True,
               'show_signup_immediately' : True,
               'extauth_email': eamap.external_email,
-               'extauth_username' : eamap.external_name.split(' ')[0],
+               'extauth_username' : eamap.external_name.replace(' ',''), # default - conjoin name, no spaces
               'extauth_name': eamap.external_name,
               }
    
--- a/common/djangoapps/student/views.py
+++ b/common/djangoapps/student/views.py
@@ -8,7 +8,6 @@ import uuid
 import feedparser
 import urllib
 import itertools
-from collections import defaultdict

 from django.conf import settings
 from django.contrib.auth import logout, authenticate, login
@@ -37,6 +36,7 @@ from xmodule.modulestore.exceptions import ItemNotFoundError
 from models import Registration, UserProfile, PendingNameChange, PendingEmailChange, CourseEnrollment
 from datetime import date
 from collections import namedtuple
+from courseware.courses import course_staff_group_name, has_staff_access_to_course, get_courses_by_university

 log = logging.getLogger("mitx.student")
 Article = namedtuple('Article', 'title url author image deck publication publish_date')
@@ -64,9 +64,9 @@ def index(request):
        from external_auth.views import edXauth_ssl_login
        return edXauth_ssl_login(request)

-    return main_index()
+    return main_index(user=request.user)

-def main_index(extra_context = {}):
+def main_index(extra_context = {}, user=None):
    '''
    Render the edX main page.

@@ -88,11 +88,8 @@ def main_index(extra_context = {}):
        entry.image = soup.img['src'] if soup.img else None
        entry.summary = soup.getText()

-    universities = defaultdict(list)
-    courses = sorted(modulestore().get_courses(), key=lambda course: course.number)
-    for course in courses:
-        universities[course.org].append(course)
-
+    # The course selection work is done in courseware.courses.
+    universities = get_courses_by_university(None)
    context = {'universities': universities, 'entries': entries}
    context.update(extra_context)
    return render_to_response('index.html', context)
@@ -184,6 +181,14 @@ def change_enrollment(request):
                      .format(user.username, enrollment.course_id))
            return {'success': False, 'error': 'The course requested does not exist.'}

+        if settings.MITX_FEATURES.get('ACCESS_REQUIRE_STAFF_FOR_COURSE'):
+            # require that user be in the staff_* group (or be an overall admin) to be able to enroll
+            # eg staff_6.002x or staff_6.00x
+            if not has_staff_access_to_course(user,course):
+                staff_group = course_staff_group_name(course)
+                log.debug('user %s denied enrollment to %s ; not in %s' % (user,course.location.url(),staff_group))
+                return {'success': False, 'error' : '%s membership required to access course.' % staff_group}
+
        enrollment, created = CourseEnrollment.objects.get_or_create(user=user, course_id=course.id)
        return {'success': True}

--- a/common/djangoapps/track/models.py
+++ b/common/djangoapps/track/models.py
@@ -1,3 +1,21 @@
 from django.db import models

-# Create your models here.
+from django.db import models
+
+class TrackingLog(models.Model):
+    dtcreated = models.DateTimeField('creation date',auto_now_add=True)
+    username = models.CharField(max_length=32,blank=True)
+    ip = models.CharField(max_length=32,blank=True)
+    event_source = models.CharField(max_length=32)
+    event_type = models.CharField(max_length=32,blank=True)
+    event = models.TextField(blank=True)
+    agent = models.CharField(max_length=256,blank=True)
+    page = models.CharField(max_length=32,blank=True,null=True)
+    time = models.DateTimeField('event time')
+
+    def __unicode__(self):
+        s = "[%s] %s@%s: %s | %s | %s | %s" % (self.time, self.username, self.ip, self.event_source,
+                                               self.event_type, self.page, self.event)
+        return s
+
+
--- a/common/djangoapps/track/views.py
+++ b/common/djangoapps/track/views.py
@@ -2,19 +2,32 @@ import json
 import logging
 import os
 import datetime
+import dateutil.parser

-# Create your views here.
+from django.contrib.auth.decorators import login_required
 from django.http import HttpResponse
 from django.http import Http404
+from django.shortcuts import redirect
 from django.conf import settings
+from mitxmako.shortcuts import render_to_response
+
+from django_future.csrf import ensure_csrf_cookie
+from track.models import TrackingLog

 log = logging.getLogger("tracking")

+LOGFIELDS = ['username','ip','event_source','event_type','event','agent','page','time']

 def log_event(event):
    event_str = json.dumps(event)
    log.info(event_str[:settings.TRACK_MAX_EVENT])
-
+    if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'):
+        event['time'] = dateutil.parser.parse(event['time'])
+        tldat = TrackingLog(**dict( (x,event[x]) for x in LOGFIELDS ))
+        try:
+            tldat.save()
+        except Exception as err:
+            log.exception(err)

 def user_track(request):
    try:  # TODO: Do the same for many of the optional META parameters
@@ -70,4 +83,16 @@ def server_track(request, event_type, event, page=None):
        "page": page,
        "time": datetime.datetime.utcnow().isoformat(),
        }
+
+    if event_type=="/event_logs" and request.user.is_staff:	# don't log
+        return
    log_event(event)
+
+@login_required
+@ensure_csrf_cookie
+def view_tracking_log(request):
+    if not request.user.is_staff:
+        return redirect('/')
+    record_instances = TrackingLog.objects.all().order_by('-time')[0:100]
+    return render_to_response('tracking_log.html',{'records':record_instances})
+
--- a/common/djangoapps/xmodule_modifiers.py
+++ b/common/djangoapps/xmodule_modifiers.py
@@ -1,9 +1,15 @@
+import re
 import json
+import logging
+
 from django.conf import settings
 from functools import wraps
 from static_replace import replace_urls
 from mitxmako.shortcuts import render_to_string
+from xmodule.seq_module import SequenceModule
+from xmodule.vertical_module import VerticalModule

+log = logging.getLogger("mitx.xmodule_modifiers")

 def wrap_xmodule(get_html, module, template):
    """
@@ -69,27 +75,31 @@ def add_histogram(get_html, module):
    the output of the old get_html function with additional information
    for admin users only, including a histogram of student answers and the
    definition of the xmodule
+
+    Does nothing if module is a SequenceModule
    """
    @wraps(get_html)
    def _get_html():
+
+        if type(module) in [SequenceModule, VerticalModule]:	# TODO: make this more general, eg use an XModule attribute instead
+            return get_html()
+
        module_id = module.id
        histogram = grade_histogram(module_id)
        render_histogram = len(histogram) > 0

-        # TODO: fixme - no filename in module.xml in general (this code block
-        # for edx4edx) the following if block is for summer 2012 edX course
-        # development; it will change when the CMS comes online
-        if settings.MITX_FEATURES.get('DISPLAY_EDIT_LINK') and settings.DEBUG and module_xml.get('filename') is not None:
-            coursename = multicourse_settings.get_coursename_from_request(request)
-            github_url = multicourse_settings.get_course_github_url(coursename)
-            fn = module_xml.get('filename')
-            if module_xml.tag=='problem': fn = 'problems/' + fn	# grrr
-            edit_link = (github_url + '/tree/master/' + fn) if github_url is not None else None
-            if module_xml.tag=='problem': edit_link += '.xml'	# grrr
+        # TODO (ichuang): Remove after fall 2012 LMS migration done
+        if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):
+            [filepath, filename] = module.definition.get('filename','')
+            osfs = module.system.filestore
+            if osfs.exists(filename):
+                filepath = filename	# if original, unmangled filename exists then use it (github doesn't like symlinks)
+            data_dir = osfs.root_path.rsplit('/')[-1]
+            edit_link = "https://github.com/MITx/%s/tree/master/%s" % (data_dir,filepath)
        else:
            edit_link = False

-        staff_context = {'definition': json.dumps(module.definition, indent=4),
+        staff_context = {'definition': module.definition.get('data'),
                         'metadata': json.dumps(module.metadata, indent=4),
                         'element_id': module.location.html_id(),
                         'edit_link': edit_link,
@@ -99,3 +109,4 @@ def add_histogram(get_html, module):
        return render_to_string("staff_problem_info.html", staff_context)

    return _get_html
+
--- a/common/lib/capa/capa/capa_problem.py
+++ b/common/lib/capa/capa/capa_problem.py
@@ -31,7 +31,7 @@ import calc
 from correctmap import CorrectMap
 import eia
 import inputtypes
-from util import contextualize_text
+from util import contextualize_text, convert_files_to_filenames

 # to be replaced with auto-registering
 import responsetypes
@@ -39,7 +39,7 @@ import responsetypes
 # dict of tagname, Response Class -- this should come from auto-registering
 response_tag_dict = dict([(x.response_tag, x) for x in responsetypes.__all__])

-entry_types = ['textline', 'schematic', 'textbox', 'imageinput', 'optioninput', 'choicegroup', 'radiogroup', 'checkboxgroup']
+entry_types = ['textline', 'schematic', 'textbox', 'imageinput', 'optioninput', 'choicegroup', 'radiogroup', 'checkboxgroup', 'filesubmission']
 solution_types = ['solution']    			# extra things displayed after "show answers" is pressed
 response_properties = ["responseparam", "answer"]    	# these get captured as student responses

@@ -228,12 +228,18 @@ class LoncapaProblem(object):

        Calls the Response for each question in this problem, to do the actual grading.
        '''
-        self.student_answers = answers
+        
+        self.student_answers = convert_files_to_filenames(answers)
+        
        oldcmap = self.correct_map				# old CorrectMap
        newcmap = CorrectMap()					# start new with empty CorrectMap
        # log.debug('Responders: %s' % self.responders)
-        for responder in self.responders.values():
-            results = responder.evaluate_answers(answers, oldcmap)      # call the responsetype instance to do the actual grading
+        for responder in self.responders.values():                  # Call each responsetype instance to do actual grading
+            if 'filesubmission' in responder.allowed_inputfields:   # File objects are passed only if responsetype 
+                                                                    #   explicitly allows for file submissions
+                results = responder.evaluate_answers(answers, oldcmap)
+            else:
+                results = responder.evaluate_answers(convert_files_to_filenames(answers), oldcmap)
            newcmap.update(results)
        self.correct_map = newcmap
        # log.debug('%s: in grade_answers, answers=%s, cmap=%s' % (self,answers,newcmap))
--- a/common/lib/capa/capa/inputtypes.py
+++ b/common/lib/capa/capa/inputtypes.py
@@ -13,6 +13,7 @@ Module containing the problem elements which render into input objects
 - checkboxgroup
 - imageinput  (for clickable image)
 - optioninput (for option list)
+- filesubmission (upload a file)

 These are matched by *.html files templates/*.html which are mako templates with the actual html.

@@ -299,6 +300,18 @@ def textline_dynamath(element, value, status, render_template, msg=''):
    return etree.XML(html)


+#-----------------------------------------------------------------------------
+@register_render_function
+def filesubmission(element, value, status, render_template, msg=''):
+    '''
+    Upload a single file (e.g. for programming assignments)
+    '''
+    eid = element.get('id')
+    context = { 'id': eid, 'state': status, 'msg': msg, 'value': value, }
+    html = render_template("filesubmission.html", context)
+    return etree.XML(html) 
+
+
 #-----------------------------------------------------------------------------
 ## TODO: Make a wrapper for <codeinput>
@register_render_function
--- a/common/lib/capa/capa/responsetypes.py
+++ b/common/lib/capa/capa/responsetypes.py
@@ -8,7 +8,6 @@ Used by capa_problem.py
 '''

 # standard library imports
-import hashlib
 import inspect
 import json
 import logging
@@ -17,7 +16,6 @@ import numpy
 import random
 import re
 import requests
-import time
 import traceback
 import abc

@@ -27,9 +25,12 @@ from correctmap import CorrectMap
 from util import *
 from lxml import etree
 from lxml.html.soupparser import fromstring as fromstring_bs	 # uses Beautiful Soup!!! FIXME?
+import xqueue_interface

 log = logging.getLogger('mitx.' + __name__)

+qinterface = xqueue_interface.XqueueInterface()
+
 #-----------------------------------------------------------------------------
 # Exceptions

@@ -162,7 +163,7 @@ class LoncapaResponse(object):
        Returns the new CorrectMap, with (correctness,msg,hint,hintmode) for each answer_id.
        '''
        new_cmap = self.get_score(student_answers)
-        self.get_hints(student_answers, new_cmap, old_cmap)
+        self.get_hints(convert_files_to_filenames(student_answers), new_cmap, old_cmap)
        # log.debug('new_cmap = %s' % new_cmap)
        return new_cmap

@@ -798,19 +799,18 @@ class SymbolicResponse(CustomResponse):

 class CodeResponse(LoncapaResponse):
    '''
-    Grade student code using an external server, called 'xqueue'
-    In contrast to ExternalResponse, CodeResponse has following behavior:
-        1) Goes through a queueing system
-        2) Does not do external request for 'get_answers'
+    Grade student code using an external queueing server, called 'xqueue'
+
+    External requests are only submitted for student submission grading 
+        (i.e. and not for getting reference answers)
    '''

    response_tag = 'coderesponse'
-    allowed_inputfields = ['textline', 'textbox']
+    allowed_inputfields = ['textbox', 'filesubmission']
    max_inputfields = 1

    def setup_response(self):
        xml = self.xml
-        self.url = xml.get('url', "http://107.20.215.194/xqueue/submit/")  # FIXME -- hardcoded url
        self.queue_name = xml.get('queuename', self.system.xqueue_default_queuename)

        answer = xml.find('answer')
@@ -849,19 +849,49 @@ class CodeResponse(LoncapaResponse):

    def get_score(self, student_answers):
        try:
-            submission = student_answers[self.answer_id]
+            submission = student_answers[self.answer_id] # Note that submission can be a file
        except Exception as err:
-            log.error('Error in CodeResponse %s: cannot get student answer for %s; student_answers=%s' % (err, self.answer_id, student_answers))
+            log.error('Error in CodeResponse %s: cannot get student answer for %s; student_answers=%s' %
+                (err, self.answer_id, convert_files_to_filenames(student_answers)))
            raise Exception(err)

-        self.context.update({'submission': submission})
-        extra_payload = {'edX_student_response': submission}
+        self.context.update({'submission': unicode(submission)})

-        r, queuekey = self._send_to_queue(extra_payload)  # TODO: Perform checks on the xqueue response
+        # Prepare xqueue request
+        #------------------------------------------------------------ 

-        # Non-null CorrectMap['queuekey'] indicates that the problem has been submitted
-        cmap = CorrectMap()
-        cmap.set(self.answer_id, queuekey=queuekey, msg='Submitted to queue')
+        # Generate header
+        queuekey = xqueue_interface.make_hashkey(self.system.seed)
+        xheader = xqueue_interface.make_xheader(lms_callback_url=self.system.xqueue_callback_url,
+                                                lms_key=queuekey,
+                                                queue_name=self.queue_name)
+
+        # Generate body
+        #   NOTE: Currently specialized to 6.00x's pyxserver, which follows the ExternalResponse interface
+        #         We should define a common interface for external code graders to CodeResponse
+        contents = {'xml': etree.tostring(self.xml, pretty_print=True),
+                    'edX_cmd': 'get_score',
+                    'edX_tests': self.tests,
+                    'processor': self.code,
+                    'edX_student_response': unicode(submission), # unicode on File object returns its filename
+                    }
+        
+        # Submit request
+        if hasattr(submission, 'read'): # Test for whether submission is a file
+            (error, msg) = qinterface.send_to_queue(header=xheader,
+                                                    body=json.dumps(contents),
+                                                    file_to_upload=submission)
+        else:
+            (error, msg) = qinterface.send_to_queue(header=xheader,
+                                                    body=json.dumps(contents))
+
+        cmap = CorrectMap() 
+        if error:
+            cmap.set(self.answer_id, queuekey=None,
+                     msg='Unable to deliver your submission to grader. (Reason: %s.) Please try again later.' % msg)
+        else:
+            # Non-null CorrectMap['queuekey'] indicates that the problem has been queued 
+            cmap.set(self.answer_id, queuekey=queuekey, msg='Submitted to grader')

        return cmap

@@ -883,17 +913,15 @@ class CodeResponse(LoncapaResponse):
            self.context['correct'][0] = admap[ad]

        # Replace 'oldcmap' with new grading results if queuekey matches.
-        #   If queuekey does not match, we keep waiting for the score_msg that will match
+        #   If queuekey does not match, we keep waiting for the score_msg whose key actually matches
        if oldcmap.is_right_queuekey(self.answer_id, queuekey):
            msg = rxml.find('message').text.replace('&nbsp;', '&#160;')
            oldcmap.set(self.answer_id, correctness=self.context['correct'][0], msg=msg, queuekey=None)  # Queuekey is consumed
        else:
-            log.debug('CodeResponse: queuekey %d does not match for answer_id=%s.' % (queuekey, self.answer_id))
+            log.debug('CodeResponse: queuekey %s does not match for answer_id=%s.' % (queuekey, self.answer_id))

        return oldcmap

-    # CodeResponse differentiates from ExternalResponse in the behavior of 'get_answers'. CodeResponse.get_answers
-    #   does NOT require a queue submission, and the answer is computed (extracted from problem XML) locally.
    def get_answers(self):
        anshtml = '<font color="blue"><span class="code-answer"><br/><pre>%s</pre><br/></span></font>' % self.answer
        return {self.answer_id: anshtml}
@@ -901,41 +929,6 @@ class CodeResponse(LoncapaResponse):
    def get_initial_display(self):
        return {self.answer_id: self.initial_display}

-    # CodeResponse._send_to_queue implements the same interface as defined for ExternalResponse's 'get_score'
-    def _send_to_queue(self, extra_payload):
-        # Prepare payload
-        xmlstr = etree.tostring(self.xml, pretty_print=True)
-        header = {'lms_callback_url': self.system.xqueue_callback_url,
-                  'queue_name': self.queue_name,
-                 }
-
-        # Queuekey generation
-        h = hashlib.md5()
-        h.update(str(self.system.seed))
-        h.update(str(time.time()))
-        queuekey = int(h.hexdigest(), 16)
-        header.update({'lms_key': queuekey})
-
-        body = {'xml': xmlstr,
-                'edX_cmd': 'get_score',
-                'edX_tests': self.tests,
-                'processor': self.code,
-                }
-        body.update(extra_payload)
-
-        payload = {'xqueue_header': json.dumps(header),
-                   'xqueue_body'  : json.dumps(body),
-                  }
-
-        # Contact queue server
-        try:
-            r = requests.post(self.url, data=payload)
-        except Exception as err:
-            msg = "Error in CodeResponse %s: cannot connect to queue server url=%s" % (err, self.url)
-            log.error(msg)
-            raise Exception(msg)
-
-        return r, queuekey

 #-----------------------------------------------------------------------------

--- a/common/lib/capa/capa/templates/filesubmission.html
+++ b/common/lib/capa/capa/templates/filesubmission.html
@@ -0,0 +1,16 @@
+<section id="filesubmission_${id}" class="filesubmission">
+    <input type="file" name="input_${id}" id="input_${id}" value="${value}" /><br />
+    % if state == 'unsubmitted':
+    <span class="unanswered" style="display:inline-block;" id="status_${id}"></span>
+    % elif state == 'correct':
+    <span class="correct" id="status_${id}"></span>
+    % elif state == 'incorrect':
+    <span class="incorrect" id="status_${id}"></span>
+    % elif state == 'incomplete':
+    <span class="incorrect" id="status_${id}"></span>
+    % endif
+    <span class="debug">(${state})</span>
+    <br/>
+    <span class="message">${msg|n}</span>
+    <br/>
+</section>
--- a/common/lib/capa/capa/util.py
+++ b/common/lib/capa/capa/util.py
@@ -30,3 +30,14 @@ def contextualize_text(text, context):  # private
    for key in sorted(context, lambda x, y: cmp(len(y), len(x))):
        text = text.replace('$' + key, str(context[key]))
    return text
+
+
+def convert_files_to_filenames(answers):
+    '''
+    Check for File objects in the dict of submitted answers,
+        convert File objects to their filename (string)
+    '''
+    new_answers = dict()
+    for answer_id in answers.keys():
+        new_answers[answer_id] = unicode(answers[answer_id])
+    return new_answers
--- a/common/lib/capa/capa/xqueue_interface.py
+++ b/common/lib/capa/capa/xqueue_interface.py
@@ -0,0 +1,114 @@
+#
+#  LMS Interface to external queueing system (xqueue)
+#
+import hashlib
+import json
+import logging
+import requests
+import time
+
+# TODO: Collection of parameters to be hooked into rest of edX system
+XQUEUE_LMS_AUTH = { 'username': 'LMS',
+                    'password': 'PaloAltoCA' }
+XQUEUE_URL = 'http://xqueue.edx.org'
+
+log = logging.getLogger('mitx.' + __name__)
+
+def make_hashkey(seed=None):
+    '''
+    Generate a string key by hashing 
+    '''
+    h = hashlib.md5()
+    if seed is not None:
+        h.update(str(seed))
+    h.update(str(time.time()))
+    return h.hexdigest()
+
+
+def make_xheader(lms_callback_url, lms_key, queue_name):
+    '''
+    Generate header for delivery and reply of queue request.
+
+    Xqueue header is a JSON-serialized dict:
+        { 'lms_callback_url': url to which xqueue will return the request (string),
+          'lms_key': secret key used by LMS to protect its state (string), 
+          'queue_name': designate a specific queue within xqueue server, e.g. 'MITx-6.00x' (string)
+        }
+    '''
+    return json.dumps({ 'lms_callback_url': lms_callback_url,
+                        'lms_key': lms_key,
+                        'queue_name': queue_name })
+
+
+def parse_xreply(xreply):
+    '''
+    Parse the reply from xqueue. Messages are JSON-serialized dict:
+        { 'return_code': 0 (success), 1 (fail)
+          'content': Message from xqueue (string)
+        }
+    '''
+    xreply = json.loads(xreply)
+    return_code = xreply['return_code']
+    content = xreply['content']
+    return (return_code, content)
+
+
+class XqueueInterface:
+    '''
+    Interface to the external grading system
+    '''
+
+    def __init__(self, url=XQUEUE_URL, auth=XQUEUE_LMS_AUTH):
+        self.url  = url
+        self.auth = auth
+        self.s = requests.session()
+        self._login()
+        
+    def send_to_queue(self, header, body, file_to_upload=None):
+        '''
+        Submit a request to xqueue.
+        
+        header: JSON-serialized dict in the format described in 'xqueue_interface.make_xheader'
+
+        body: Serialized data for the receipient behind the queueing service. The operation of
+                xqueue is agnostic to the contents of 'body'
+
+        file_to_upload: File object to be uploaded to xqueue along with queue request
+
+        Returns (error_code, msg) where error_code != 0 indicates an error
+        '''
+        # Attempt to send to queue
+        (error, msg) = self._send_to_queue(header, body, file_to_upload)
+
+        if error and (msg == 'login_required'): # Log in, then try again
+            self._login()
+            (error, msg) = self._send_to_queue(header, body, file_to_upload)
+
+        return (error, msg)
+
+    def _login(self):
+        try:
+            r = self.s.post(self.url+'/xqueue/login/', data={ 'username': self.auth['username'],
+                                                              'password': self.auth['password'] })
+        except requests.exceptions.ConnectionError, err:
+            log.error(err)
+            return (1, 'cannot connect to server') 
+
+        return parse_xreply(r.text)
+
+    def _send_to_queue(self, header, body, file_to_upload=None):
+
+        payload = {'xqueue_header': header,
+                   'xqueue_body'  : body}
+
+        files = None
+        if file_to_upload is not None:
+            files = { file_to_upload.name: file_to_upload }
+
+        try:
+            r = self.s.post(self.url+'/xqueue/submit/', data=payload, files=files)
+        except requests.exceptions.ConnectionError, err:
+            log.error(err)
+            return (1, 'cannot connect to server') 
+
+        return parse_xreply(r.text)
--- a/common/lib/xmodule/xmodule/capa_module.py
+++ b/common/lib/xmodule/xmodule/capa_module.py
@@ -17,6 +17,7 @@ from xmodule.exceptions import NotFoundError
 from progress import Progress
 from capa.capa_problem import LoncapaProblem
 from capa.responsetypes import StudentInputError
+from capa.util import convert_files_to_filenames

 log = logging.getLogger("mitx.courseware")

@@ -425,10 +426,9 @@ class CapaModule(XModule):
        event_info = dict()
        event_info['state'] = self.lcp.get_state()
        event_info['problem_id'] = self.location.url()
-
+         
        answers = self.make_dict_of_responses(get)
-
-        event_info['answers'] = answers
+        event_info['answers'] = convert_files_to_filenames(answers)

        # Too late. Cannot submit
        if self.closed():
@@ -436,8 +436,7 @@ class CapaModule(XModule):
            self.system.track_function('save_problem_check_fail', event_info)
            raise NotFoundError('Problem is closed')

-        # Problem submitted. Student should reset before checking
-        # again.
+        # Problem submitted. Student should reset before checking again
        if self.lcp.done and self.rerandomize == "always":
            event_info['failure'] = 'unreset'
            self.system.track_function('save_problem_check_fail', event_info)
--- a/common/lib/xmodule/xmodule/html_module.py
+++ b/common/lib/xmodule/xmodule/html_module.py
@@ -94,7 +94,15 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
                        msg = "Couldn't parse html in {0}.".format(filepath)
                        log.warning(msg)
                        system.error_tracker("Warning: " + msg)
-                    return {'data' : html}
+
+                    definition = {'data' : html}
+
+                    # TODO (ichuang): remove this after migration
+                    # for Fall 2012 LMS migration: keep filename (and unmangled filename)
+                    definition['filename'] = [ filepath, filename ]
+
+                    return definition
+
            except (ResourceNotFoundError) as err:
                msg = 'Unable to load file contents at path {0}: {1} '.format(
                    filepath, err)
--- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee
+++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee
@@ -13,7 +13,8 @@ class @Problem
    MathJax.Hub.Queue ["Typeset", MathJax.Hub]
    window.update_schematics()
    @$('section.action input:button').click @refreshAnswers
-    @$('section.action input.check').click @check
+    @$('section.action input.check').click @check_fd
+    #@$('section.action input.check').click @check
    @$('section.action input.reset').click @reset
    @$('section.action input.show').click @show
    @$('section.action input.save').click @save
@@ -45,6 +46,51 @@ class @Problem
      $('head')[0].appendChild(s[0])
      $(placeholder).remove()

+  ###
+  # 'check_fd' uses FormData to allow file submissions in the 'problem_check' dispatch,
+  #      in addition to simple querystring-based answers
+  #
+  # NOTE: The dispatch 'problem_check' is being singled out for the use of FormData;
+  #       maybe preferable to consolidate all dispatches to use FormData
+  ###
+  check_fd: =>
+    Logger.log 'problem_check', @answers
+
+    # If there are no file inputs in the problem, we can fall back on @check
+    if $('input:file').length == 0 
+      @check()
+      return
+
+    if not window.FormData
+      alert "Sorry, your browser does not support file uploads. Your submit request could not be fulfilled. If you can, please use Chrome or Safari which have been verified to support file uploads."
+      return
+
+    fd = new FormData()
+
+    @$("[id^=input_#{@element_id.replace(/problem_/, '')}_]").each (index, element) ->
+      if element.type is 'file'
+        if element.files[0] instanceof File
+          fd.append(element.id, element.files[0])
+        else
+          fd.append(element.id, '')
+      else
+        fd.append(element.id, element.value)
+
+    settings = 
+      type: "POST"
+      data: fd
+      processData: false
+      contentType: false
+      success: (response) => 
+        switch response.success
+          when 'incorrect', 'correct'
+            @render(response.contents)
+            @updateProgress response
+          else
+            alert(response.success)
+
+    $.ajaxWithPrefix("#{@url}/problem_check", settings)
+
  check: =>
    Logger.log 'problem_check', @answers
    $.postWithPrefix "#{@url}/problem_check", @answers, (response) =>
--- a/common/lib/xmodule/xmodule/modulestore/xml.py
+++ b/common/lib/xmodule/xmodule/modulestore/xml.py
@@ -146,19 +146,30 @@ class XMLModuleStore(ModuleStoreBase):
                           os.path.exists(self.data_dir / d / "course.xml")]

        for course_dir in course_dirs:
-            try:
-                # Special-case code here, since we don't have a location for the
-                # course before it loads.
-                # So, make a tracker to track load-time errors, then put in the right
-                # place after the course loads and we have its location
-                errorlog = make_error_tracker()
-                course_descriptor = self.load_course(course_dir, errorlog.tracker)
-                self.courses[course_dir] = course_descriptor
-                self._location_errors[course_descriptor.location] = errorlog
-            except:
-                msg = "Failed to load course '%s'" % course_dir
-                log.exception(msg)
+            self.try_load_course(course_dir)

+    def try_load_course(self,course_dir):
+        '''
+        Load a course, keeping track of errors as we go along.
+        '''
+        try:
+            # Special-case code here, since we don't have a location for the
+            # course before it loads.
+            # So, make a tracker to track load-time errors, then put in the right
+            # place after the course loads and we have its location
+            errorlog = make_error_tracker()
+            course_descriptor = self.load_course(course_dir, errorlog.tracker)
+            self.courses[course_dir] = course_descriptor
+            self._location_errors[course_descriptor.location] = errorlog
+        except:
+            msg = "Failed to load course '%s'" % course_dir
+            log.exception(msg)
+
+    def __unicode__(self):
+        '''
+        String representation - for debugging
+        '''
+        return '<XMLModuleStore>data_dir=%s, %d courses, %d modules' % (self.data_dir,len(self.courses),len(self.modules))

    def load_course(self, course_dir, tracker):
        """
--- a/common/lib/xmodule/xmodule/x_module.py
+++ b/common/lib/xmodule/xmodule/x_module.py
@@ -204,6 +204,8 @@ class XModule(HTMLSnippet):
        '''
        return self.metadata.get('display_name',
                                 self.url_name.replace('_', ' '))
+    def __unicode__(self):
+        return '<x_module(name=%s, category=%s, id=%s)>' % (self.name, self.category, self.id)

    def get_children(self):
        '''
--- a/common/lib/xmodule/xmodule/xml_module.py
+++ b/common/lib/xmodule/xmodule/xml_module.py
@@ -41,6 +41,7 @@ class XmlDescriptor(XModuleDescriptor):
    # to definition_from_xml, and from the xml returned by definition_to_xml
    metadata_attributes = ('format', 'graceperiod', 'showanswer', 'rerandomize',
        'start', 'due', 'graded', 'display_name', 'url_name', 'hide_from_toc',
+        'ispublic', 	# if True, then course is listed for all users; see 
        # VS[compat] Remove once unused.
        'name', 'slug')

@@ -109,6 +110,7 @@ class XmlDescriptor(XModuleDescriptor):
        filename = xml_object.get('filename')
        if filename is None:
            definition_xml = copy.deepcopy(xml_object)
+            filepath = ''
        else:
            filepath = cls._format_filepath(xml_object.tag, filename)

@@ -136,7 +138,13 @@ class XmlDescriptor(XModuleDescriptor):
                raise Exception, msg, sys.exc_info()[2]

        cls.clean_metadata_from_xml(definition_xml)
-        return cls.definition_from_xml(definition_xml, system)
+        definition = cls.definition_from_xml(definition_xml, system)
+
+        # TODO (ichuang): remove this after migration
+        # for Fall 2012 LMS migration: keep filename (and unmangled filename)
+        definition['filename'] = [ filepath, filename ]
+
+        return definition


    @classmethod
--- a/lms/djangoapps/courseware/courses.py
+++ b/lms/djangoapps/courseware/courses.py
@@ -1,3 +1,4 @@
+from collections import defaultdict
 from fs.errors import ResourceNotFoundError
 from functools import wraps
 import logging
@@ -114,3 +115,57 @@ def get_course_info_section(course, section_key):
            return "! Info section missing !"

    raise KeyError("Invalid about key " + str(section_key))
+
+def course_staff_group_name(course):
+    '''
+    course should be either a CourseDescriptor instance, or a string (the .course entry of a Location)
+    '''
+    if isinstance(course,str):
+        coursename = course
+    else:
+        coursename = course.metadata.get('data_dir','UnknownCourseName')
+    if not coursename:					# Fall 2012: not all course.xml have metadata correct yet
+        coursename = course.metadata.get('course','')
+    return 'staff_%s' % coursename
+
+def has_staff_access_to_course(user,course):
+    '''
+    Returns True if the given user has staff access to the course.
+    This means that user is in the staff_* group, or is an overall admin.
+    '''
+    if user is None or (not user.is_authenticated()) or course is None:
+        return False
+    if user.is_staff:
+        return True
+    user_groups = [x[1] for x in user.groups.values_list()]	# note this is the Auth group, not UserTestGroup
+    staff_group = course_staff_group_name(course)
+    log.debug('course %s user %s groups %s' % (staff_group, user, user_groups))
+    if staff_group in user_groups:
+        return True
+    return False
+
+def has_access_to_course(user,course):
+    if course.metadata.get('ispublic'):
+        return True
+    return has_staff_access_to_course(user,course)
+
+def get_courses_by_university(user):
+    '''
+    Returns dict of lists of courses available, keyed by course.org (ie university).
+    Courses are sorted by course.number.
+
+    if ACCESS_REQUIRE_STAFF_FOR_COURSE then list only includes those accessible to user.
+    '''
+    # TODO: Clean up how 'error' is done.
+    # filter out any courses that errored.
+    courses = [c for c in modulestore().get_courses()
+               if isinstance(c, CourseDescriptor)]
+    courses = sorted(courses, key=lambda course: course.number)
+    universities = defaultdict(list)
+    for course in courses:
+        if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):
+            if not has_access_to_course(user,course):
+                continue
+        universities[course.org].append(course)
+    return universities
+    
--- a/lms/djangoapps/courseware/module_render.py
+++ b/lms/djangoapps/courseware/module_render.py
@@ -15,6 +15,8 @@ from xmodule.exceptions import NotFoundError
 from xmodule.x_module import ModuleSystem
 from xmodule_modifiers import replace_static_urls, add_histogram, wrap_xmodule

+from courseware.courses import has_staff_access_to_course
+
 log = logging.getLogger("mitx.courseware")


@@ -188,8 +190,9 @@ def get_module(user, request, location, student_module_cache, position=None):
        module.metadata['data_dir']
    )

-    if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF') and user.is_staff:
-        module.get_html = add_histogram(module.get_html, module)
+    if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF'):
+        if has_staff_access_to_course(user, module.location.course):
+            module.get_html = add_histogram(module.get_html, module)

    return module

@@ -306,10 +309,15 @@ def modx_dispatch(request, dispatch=None, id=None):
      - id -- the module id. Used to look up the XModule instance
    '''
    # ''' (fix emacs broken parsing)
+    # Check for submitted files
+    p = request.POST.copy()
+    if request.FILES:
+        for inputfile_id in request.FILES.keys():
+            p[inputfile_id] = request.FILES[inputfile_id]

    student_module_cache = StudentModuleCache.cache_for_descriptor_descendents(request.user, modulestore().get_item(id))
    instance = get_module(request.user, request, id, student_module_cache)
-    
+
    instance_module = get_instance_module(request.user, instance, student_module_cache)
    shared_module = get_shared_instance_module(request.user, instance, student_module_cache)
    
@@ -321,7 +329,7 @@ def modx_dispatch(request, dispatch=None, id=None):

    # Let the module handle the AJAX
    try:
-        ajax_return = instance.handle_ajax(dispatch, request.POST)
+        ajax_return = instance.handle_ajax(dispatch, p)
    except NotFoundError:
        log.exception("Module indicating to user that request doesn't exist")
        raise Http404
--- a/lms/djangoapps/courseware/views.py
+++ b/lms/djangoapps/courseware/views.py
@@ -1,4 +1,3 @@
-from collections import defaultdict
 import json
 import logging
 import urllib
@@ -28,7 +27,7 @@ from xmodule.course_module import CourseDescriptor
 from util.cache import cache, cache_if_anonymous
 from student.models import UserTestGroup, CourseEnrollment
 from courseware import grades
-from courseware.courses import check_course
+from courseware.courses import check_course, get_courses_by_university


 log = logging.getLogger("mitx.courseware")
@@ -59,19 +58,12 @@ def user_groups(user):
@ensure_csrf_cookie
@cache_if_anonymous
 def courses(request):
-    # TODO: Clean up how 'error' is done.
-
-    # filter out any courses that errored.
-    courses = [c for c in modulestore().get_courses()
-               if isinstance(c, CourseDescriptor)]
-    courses = sorted(courses, key=lambda course: course.number)
-    universities = defaultdict(list)
-    for course in courses:
-        universities[course.org].append(course)
-
+    '''
+    Render "find courses" page.  The course selection work is done in courseware.courses.
+    '''
+    universities = get_courses_by_university(request.user)
    return render_to_response("courses.html", {'universities': universities})

-
@cache_control(no_cache=True, no_store=True, must_revalidate=True)
 def gradebook(request, course_id):
    if 'course_admin' not in user_groups(request.user):
@@ -155,6 +147,7 @@ def render_accordion(request, course, chapter, section):
    return render_to_string('accordion.html', context)


+@login_required
@ensure_csrf_cookie
@cache_control(no_cache=True, no_store=True, must_revalidate=True)
 def index(request, course_id, chapter=None, section=None,
@@ -177,6 +170,11 @@ def index(request, course_id, chapter=None, section=None,
     - HTTPresponse
    '''
    course = check_course(course_id)
+    registered = registered_for_course(course, request.user)
+    if not registered:
+        log.debug('User %s tried to view course %s but is not enrolled' % (request.user,course.location.url()))
+        return redirect(reverse('about_course', args=[course.id]))
+
    try:
        context = {
            'csrf': csrf(request)['csrf_token'],
@@ -271,14 +269,18 @@ def course_info(request, course_id):
    return render_to_response('info.html', {'course': course})


+def registered_for_course(course, user):
+    '''Return CourseEnrollment if user is registered for course, else False'''
+    if user is None:
+        return False
+    if user.is_authenticated():
+        return CourseEnrollment.objects.filter(user=user, course_id=course.id).exists()
+    else:
+        return False
+
@ensure_csrf_cookie
@cache_if_anonymous
 def course_about(request, course_id):
-    def registered_for_course(course, user):
-        if user.is_authenticated():
-            return CourseEnrollment.objects.filter(user=user, course_id=course.id).exists()
-        else:
-            return False
    course = check_course(course_id, course_must_be_open=False)
    registered = registered_for_course(course, request.user)
    return render_to_response('portal/course_about.html', {'course': course, 'registered': registered})
@@ -293,7 +295,7 @@ def university_profile(request, org_id):
        raise Http404("University Profile not found for {0}".format(org_id))

    # Only grab courses for this org...
-    courses = [c for c in all_courses if c.org == org_id]
+    courses = get_courses_by_university(request.user)[org_id]
    context = dict(courses=courses, org_id=org_id)
    template_file = "university_profile/{0}.html".format(org_id).lower()

--- a/lms/djangoapps/lms_migration/init.py
+++ b/lms/djangoapps/lms_migration/init.py
--- a/lms/djangoapps/lms_migration/migrate.py
+++ b/lms/djangoapps/lms_migration/migrate.py
@@ -0,0 +1,110 @@
+#
+# migration tools for content team to go from stable-edx4edx to LMS+CMS
+#
+
+import logging
+from pprint import pprint
+import xmodule.modulestore.django as xmodule_django
+from xmodule.modulestore.django import modulestore
+
+from django.http import HttpResponse
+from django.conf import settings
+
+log = logging.getLogger("mitx.lms_migrate")
+LOCAL_DEBUG = True
+ALLOWED_IPS = settings.LMS_MIGRATION_ALLOWED_IPS
+
+def escape(s):
+    """escape HTML special characters in string"""
+    return str(s).replace('<','&lt;').replace('>','&gt;')
+
+def manage_modulestores(request,reload_dir=None):
+    '''
+    Manage the static in-memory modulestores.
+
+    If reload_dir is not None, then instruct the xml loader to reload that course directory.
+    '''
+    html = "<html><body>"
+
+    def_ms = modulestore()
+    courses = def_ms.get_courses()
+
+    #----------------------------------------
+    # check on IP address of requester
+
+    ip = request.META.get('HTTP_X_REAL_IP','')	# nginx reverse proxy
+    if not ip:
+        ip = request.META.get('REMOTE_ADDR','None')
+    
+    if LOCAL_DEBUG:
+        html += '<h3>IP address: %s ' % ip
+        html += '<h3>User: %s ' % request.user
+        log.debug('request from ip=%s, user=%s' % (ip,request.user))
+
+    if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
+        if request.user and request.user.is_staff:
+            log.debug('request allowed because user=%s is staff' % request.user)
+        else:
+            html += 'Permission denied'
+            html += "</body></html>"
+            log.debug('request denied, ALLOWED_IPS=%s' % ALLOWED_IPS)
+            return HttpResponse(html)    
+
+    #----------------------------------------
+    # reload course if specified
+
+    if reload_dir is not None:
+        if reload_dir not in def_ms.courses:
+            html += "<h2><font color='red'>Error: '%s' is not a valid course directory</font></h2>" % reload_dir
+        else:
+            html += "<h2><font color='blue'>Reloaded course directory '%s'</font></h2>" % reload_dir
+            def_ms.try_load_course(reload_dir)
+
+    #----------------------------------------
+
+    html += '<h2>Courses loaded in the modulestore</h2>'
+    html += '<ol>'
+    for cdir, course in def_ms.courses.items():
+        html += '<li><a href="%s/migrate/reload/%s">%s</a> (%s)</li>' % (settings.MITX_ROOT_URL,
+                                                            escape(cdir),
+                                                            escape(cdir),
+                                                            course.location.url())
+    html += '</ol>'
+
+    #----------------------------------------
+
+    dumpfields = ['definition','location','metadata']
+    
+    for cdir, course in def_ms.courses.items():
+        html += '<hr width="100%"/>'
+        html += '<h2>Course: %s (%s)</h2>' % (course.metadata['display_name'],cdir)
+
+        for field in dumpfields:
+            data = getattr(course,field)
+            html += '<h3>%s</h3>' % field
+            if type(data)==dict:
+                html += '<ul>'
+                for k,v in data.items():
+                    html += '<li>%s:%s</li>' % (escape(k),escape(v))
+                html += '</ul>'
+            else:
+                html += '<ul><li>%s</li></ul>' % escape(data)
+        
+
+    #----------------------------------------
+
+    html += '<hr width="100%"/>'
+    html += "courses: <pre>%s</pre>" % escape(courses)
+
+    ms = xmodule_django._MODULESTORES
+    html += "modules: <pre>%s</pre>" % escape(ms)
+    html += "default modulestore: <pre>%s</pre>" % escape(unicode(def_ms))
+
+    #----------------------------------------
+
+    log.debug('_MODULESTORES=%s' % ms)
+    log.debug('courses=%s' % courses)
+    log.debug('def_ms=%s' % unicode(def_ms))
+
+    html += "</body></html>"
+    return HttpResponse(html)    
--- a/lms/envs/common.py
+++ b/lms/envs/common.py
@@ -48,6 +48,17 @@ MITX_FEATURES = {
    ## DO NOT SET TO True IN THIS FILE
    ## Doing so will cause all courses to be released on production
    'DISABLE_START_DATES': False,  # When True, all courses will be active, regardless of start date
+
+    'ENABLE_TEXTBOOK' : True,
+    'ENABLE_DISCUSSION' : True,
+
+    'ENABLE_SQL_TRACKING_LOGS': False,
+    'ENABLE_LMS_MIGRATION': False,
+
+    # extrernal access methods
+    'ACCESS_REQUIRE_STAFF_FOR_COURSE': False,
+    'AUTH_USE_OPENID': False,
+    'AUTH_USE_MIT_CERTIFICATES' : False,
 }

 # Used for A/B testing
--- a/lms/envs/dev.py
+++ b/lms/envs/dev.py
@@ -14,6 +14,7 @@ DEBUG = True
 TEMPLATE_DEBUG = True

 MITX_FEATURES['DISABLE_START_DATES'] = True
+MITX_FEATURES['ENABLE_SQL_TRACKING_LOGS'] = True

 WIKI_ENABLED = True

@@ -58,6 +59,12 @@ CACHE_TIMEOUT = 0
 # Dummy secret key for dev
 SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'

+################################ LMS Migration #################################
+MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True
+MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = False   # require that user be in the staff_* group to be able to enroll
+
+LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1']
+
 ################################ OpenID Auth #################################
 MITX_FEATURES['AUTH_USE_OPENID'] = True
 MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True
--- a/lms/envs/dev_ike.py
+++ b/lms/envs/dev_ike.py
@@ -9,108 +9,10 @@ sessions. Assumes structure:
 """
 from .common import *
 from .logsettings import get_logger_config
+from .dev import *

-DEBUG = True
-TEMPLATE_DEBUG = True
+WIKI_ENABLED = False
+MITX_FEATURES['ENABLE_TEXTBOOK'] = False
+MITX_FEATURES['ENABLE_DISCUSSION'] = False
+MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True	  # require that user be in the staff_* group to be able to enroll

-MITX_FEATURES['DISABLE_START_DATES'] = True
-
-WIKI_ENABLED = True
-
-LOGGING = get_logger_config(ENV_ROOT / "log", 
-                            logging_env="dev",
-                            tracking_filename="tracking.log",
-                            debug=True)
-
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': ENV_ROOT / "db" / "mitx.db",
-    }
-}
-
-CACHES = {
-    # This is the cache used for most things. Askbot will not work without a 
-    # functioning cache -- it relies on caching to load its settings in places.
-    # In staging/prod envs, the sessions also live here.
-    'default': {
-        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
-        'LOCATION': 'mitx_loc_mem_cache',
-        'KEY_FUNCTION': 'util.memcache.safe_key',
-    },
-
-    # The general cache is what you get if you use our util.cache. It's used for
-    # things like caching the course.xml file for different A/B test groups.
-    # We set it to be a DummyCache to force reloading of course.xml in dev.
-    # In staging environments, we would grab VERSION from data uploaded by the
-    # push process.
-    'general': {
-        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
-        'KEY_PREFIX': 'general',
-        'VERSION': 4,
-        'KEY_FUNCTION': 'util.memcache.safe_key',
-    }
-}
-
-# Dummy secret key for dev
-SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'
-
-################################ OpenID Auth #################################
-MITX_FEATURES['AUTH_USE_OPENID'] = True
-
-INSTALLED_APPS += ('external_auth',) 
-INSTALLED_APPS += ('django_openid_auth',)
-#INSTALLED_APPS += ('ssl_auth',)
-
-#MIDDLEWARE_CLASSES += (
-#    #'ssl_auth.ssl_auth.NginxProxyHeaderMiddleware',		# ssl authentication behind nginx proxy
-#    )
-
-#AUTHENTICATION_BACKENDS = (
-#    'django_openid_auth.auth.OpenIDBackend',
-#    'django.contrib.auth.backends.ModelBackend',
-#    )
-
-OPENID_CREATE_USERS = False
-OPENID_UPDATE_DETAILS_FROM_SREG = True
-OPENID_SSO_SERVER_URL = 'https://www.google.com/accounts/o8/id'
-OPENID_USE_AS_ADMIN_LOGIN = False
-#import external_auth.views as edXauth
-#OPENID_RENDER_FAILURE = edXauth.edXauth_openid
-
-################################ DEBUG TOOLBAR #################################
-INSTALLED_APPS += ('debug_toolbar',) 
-MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',)
-INTERNAL_IPS = ('127.0.0.1',)
-
-DEBUG_TOOLBAR_PANELS = (
-   'debug_toolbar.panels.version.VersionDebugPanel',
-   'debug_toolbar.panels.timer.TimerDebugPanel',
-   'debug_toolbar.panels.settings_vars.SettingsVarsDebugPanel',
-   'debug_toolbar.panels.headers.HeaderDebugPanel',
-   'debug_toolbar.panels.request_vars.RequestVarsDebugPanel',
-   'debug_toolbar.panels.sql.SQLDebugPanel',
-   'debug_toolbar.panels.signals.SignalDebugPanel',
-   'debug_toolbar.panels.logger.LoggingPanel',
-
-#  Enabling the profiler has a weird bug as of django-debug-toolbar==0.9.4 and
-#  Django=1.3.1/1.4 where requests to views get duplicated (your method gets 
-#  hit twice). So you can uncomment when you need to diagnose performance 
-#  problems, but you shouldn't leave it on.
-#  'debug_toolbar.panels.profiling.ProfilingDebugPanel',
-)
-
-############################ FILE UPLOADS (ASKBOT) #############################
-DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
-MEDIA_ROOT = ENV_ROOT / "uploads"
-MEDIA_URL = "/static/uploads/"
-STATICFILES_DIRS.append(("uploads", MEDIA_ROOT))
-FILE_UPLOAD_TEMP_DIR = ENV_ROOT / "uploads"
-FILE_UPLOAD_HANDLERS = (
-    'django.core.files.uploadhandler.MemoryFileUploadHandler',
-    'django.core.files.uploadhandler.TemporaryFileUploadHandler',
-)
-
-########################### PIPELINE #################################
-
-PIPELINE_SASS_ARGUMENTS = '-r {proj_dir}/static/sass/bourbon/lib/bourbon.rb'.format(proj_dir=PROJECT_ROOT)
--- a/lms/templates/course_navigation.html
+++ b/lms/templates/course_navigation.html
@@ -14,10 +14,16 @@ def url_class(url):
      <li class="courseware"><a href="${reverse('courseware', args=[course.id])}" class="${url_class('courseware')}">Courseware</a></li>
      <li class="info"><a href="${reverse('info', args=[course.id])}" class="${url_class('info')}">Course Info</a></li>
 % if user.is_authenticated():
+% if settings.MITX_FEATURES.get('ENABLE_TEXTBOOK'):
      <li class="book"><a href="${reverse('book', args=[course.id])}" class="${url_class('book')}">Textbook</a></li>
+% endif 
+% if settings.MITX_FEATURES.get('ENABLE_DISCUSSION'):
      <li class="discussion"><a href="${reverse('questions')}">Discussion</a></li>
+% endif 
 % endif
+% if settings.WIKI_ENABLED:
      <li class="wiki"><a href="${reverse('wiki_root', args=[course.id])}" class="${url_class('wiki')}">Wiki</a></li>
+% endif
 % if user.is_authenticated():
      <li class="profile"><a href="${reverse('profile', args=[course.id])}" class="${url_class('profile')}">Profile</a></li>
 % endif
--- a/lms/templates/portal/course_about.html
+++ b/lms/templates/portal/course_about.html
@@ -19,6 +19,8 @@
        $(document).delegate('#class_enroll_form', 'ajax:success', function(data, json, xhr) {
          if(json.success) {
            location.href="${reverse('dashboard')}";
+          }else{
+	    $('#register_message).html("<p><font color='red'>" + json.error + "</font></p>")
          }
        });
      })(this)
@@ -60,9 +62,24 @@
        <div class="main-cta">
          %if user.is_authenticated():
            %if registered:
+   	        <%
+		  ## TODO: move this logic into a view
+                  if course.has_started() or settings.MITX_FEATURES['DISABLE_START_DATES']:
+                      course_target = reverse('info', args=[course.id])
+                  else:
+                      course_target = reverse('about_course', args=[course.id])
+                  show_link = settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION')
+                %>
+	        %if show_link:
+                <a href="${course_target}">
+                %endif
                <span class="register disabled">You are registered for this course (${course.number}).</span>
+	        %if show_link:
+                </a>
+                %endif
            %else:
                <a href="#" class="register">Register for ${course.number}</a>
+	        <div id="register_message"></div>
            %endif
          %else:
            <a href="#signup-modal" class="register" rel="leanModal" data-notice='You must Sign Up or <a href="#login-modal" rel="leanModal">Log In</a> to enroll.'>Register for ${course.number}</a>
--- a/lms/templates/staff_problem_info.html
+++ b/lms/templates/staff_problem_info.html
@@ -1,11 +1,11 @@
 ${module_content}
-<div class="staff_info">
-definition = ${definition | h}
-metadata = ${metadata | h}
-</div>
 %if edit_link:
 <div><a href="${edit_link}">Edit</a></div>
 % endif
+<div class="staff_info">
+definition = <pre>${definition | h}</pre>
+metadata = ${metadata | h}
+</div>
 %if render_histogram:
 <div id="histogram_${element_id}" class="histogram" data-histogram="${histogram}"></div>
 %endif
--- a/lms/templates/tracking_log.html
+++ b/lms/templates/tracking_log.html
@@ -0,0 +1,14 @@
+<html>
+<h1>Tracking Log</h1>
+<table border="1"><tr><th>datetime</th><th>username</th><th>ipaddr</th><th>source</th><th>type</th></tr>
+% for rec in records:
+    <tr>
+    <td>${rec.time}</td> 
+    <td>${rec.username}</td>  
+    <td>${rec.ip}</td>  
+    <td>${rec.event_source}</td>   
+    <td>${rec.event_type}</td>   
+    </tr>
+% endfor
+</table>    
+</html>
--- a/lms/urls.py
+++ b/lms/urls.py
@@ -172,6 +172,17 @@ if settings.MITX_FEATURES.get('AUTH_USE_OPENID'):
        url(r'^openid/logo.gif$', 'django_openid_auth.views.logo', name='openid-logo'),
        )

+if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):
+    urlpatterns += (
+        url(r'^migrate/modules$', 'lms_migration.migrate.manage_modulestores'),
+        url(r'^migrate/reload/(?P<reload_dir>[^/]+)$', 'lms_migration.migrate.manage_modulestores'),
+        )
+
+if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'):
+    urlpatterns += (
+        url(r'^event_logs$', 'track.views.view_tracking_log'),
+        )
+
 urlpatterns = patterns(*urlpatterns)

 if settings.DEBUG:
--- a/utility-scripts/create_groups.py
+++ b/utility-scripts/create_groups.py
@@ -0,0 +1,35 @@
+#!/usr/bin/python
+#
+# File:   create_groups.py
+#
+# Create all staff_* groups for classes in data directory.
+
+import os, sys, string, re
+
+sys.path.append(os.path.abspath('.'))
+os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev'
+
+try:
+    from lms.envs.dev import *
+except Exception as err:
+    print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory."
+    sys.exit(-1)
+
+from django.conf import settings
+from django.contrib.auth.models import User, Group
+from path import path
+
+data_dir = settings.DATA_DIR
+print "data_dir = %s" % data_dir
+
+for course_dir in os.listdir(data_dir):
+    # print course_dir
+    if not os.path.isdir(path(data_dir) / course_dir):
+        continue
+    gname = 'staff_%s' % course_dir
+    if Group.objects.filter(name=gname):
+        print "group exists for %s" % gname
+        continue
+    g = Group(name=gname)
+    g.save()
+    print "created group %s" % gname
--- a/utility-scripts/create_user.py
+++ b/utility-scripts/create_user.py
@@ -0,0 +1,149 @@
+#!/usr/bin/python
+#
+# File:   create_user.py
+#
+# Create user.  Prompt for groups and ExternalAuthMap
+
+import os, sys, string, re
+import datetime
+from getpass import getpass
+import json
+import readline
+
+sys.path.append(os.path.abspath('.'))
+os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev'
+
+try:
+    from lms.envs.dev import *
+except Exception as err:
+    print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory."
+    sys.exit(-1)
+
+from student.models import UserProfile, Registration
+from external_auth.models import ExternalAuthMap
+from django.contrib.auth.models import User, Group
+from random import choice
+
+class MyCompleter(object):  # Custom completer
+
+    def __init__(self, options):
+        self.options = sorted(options)
+
+    def complete(self, text, state):
+        if state == 0:  # on first trigger, build possible matches
+            if text:  # cache matches (entries that start with entered text)
+                self.matches = [s for s in self.options 
+                                    if s and s.startswith(text)]
+            else:  # no text entered, all matches possible
+                self.matches = self.options[:]
+
+        # return match indexed by state
+        try: 
+            return self.matches[state]
+        except IndexError:
+            return None
+
+def GenPasswd(length=8, chars=string.letters + string.digits):
+    return ''.join([choice(chars) for i in range(length)])
+
+#-----------------------------------------------------------------------------
+# main
+
+while True:
+    uname = raw_input('username: ')
+    if User.objects.filter(username=uname):
+        print "username %s already taken" % uname
+    else:
+        break
+
+make_eamap = False
+if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y':
+    email = '%s@MIT.EDU' % uname
+    if not email.endswith('@MIT.EDU'):
+        print "Failed - email must be @MIT.EDU"
+        sys.exit(-1)
+    mit_domain = 'ssl:MIT'
+    if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain):
+        print "Failed - email %s already exists as external_id" % email
+        sys.exit(-1)
+    make_eamap = True
+    password = GenPasswd(12)
+    
+    # get name from kerberos
+    kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip()
+    name = raw_input('Full name: [%s] ' % kname).strip()
+    if name=='':
+        name = kname
+    print "name = %s" % name
+else:
+    while True:
+        password = getpass()
+        password2 = getpass()
+        if password == password2:
+            break
+        print "Oops, passwords do not match, please retry"
+
+    while True:
+        email = raw_input('email: ')
+        if User.objects.filter(email=email):
+            print "email %s already taken" % email
+        else:
+            break
+        
+    name = raw_input('Full name: ')
+
+        
+user = User(username=uname, email=email, is_active=True)
+user.set_password(password)
+try:
+    user.save()
+except IntegrityError:
+    print "Oops, failed to create user %s, IntegrityError" % user
+    raise
+    
+r = Registration()
+r.register(user)
+
+up = UserProfile(user=user)
+up.name = name
+up.save()
+
+if make_eamap:
+    credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email)
+    eamap = ExternalAuthMap(external_id = email,
+                            external_email = email,
+                            external_domain = mit_domain,
+                            external_name = name,
+                            internal_password = password,
+                            external_credentials = json.dumps(credentials),
+        )
+    eamap.user = user
+    eamap.dtsignup = datetime.datetime.now()
+    eamap.save()
+
+print "User %s created successfully!" % user
+
+if not raw_input('Add user %s to any groups? [n] ' % user).lower()=='y':
+    sys.exit(0)
+
+print "Here are the groups available:"
+
+groups = [str(g.name) for g in Group.objects.all()]
+print groups
+
+completer = MyCompleter(groups)
+readline.set_completer(completer.complete)
+readline.parse_and_bind('tab: complete')
+
+while True:
+    gname = raw_input("Add group (tab to autocomplete, empty line to end): ")
+    if not gname:
+        break
+    if not gname in groups:
+        print "Unknown group %s" % gname
+        continue
+    g = Group.objects.get(name=gname)
+    user.groups.add(g)
+    print "Added %s to group %s" % (user,g)
+
+print "Done!"