Merge pull request #342 from MITx/lms-migration

LMS migration

common/djangoapps/external_auth/views.py

@@ -150,7 +150,7 @@ def edXauth_signup(request, eamap=None):
     context = {'has_extauth_info': True,
                'show_signup_immediately' : True,
                'extauth_email': eamap.external_email,
-               'extauth_username' : eamap.external_name.split(' ')[0],
+               'extauth_username' : eamap.external_name.replace(' ',''), # default - conjoin name, no spaces
                'extauth_name': eamap.external_name,
                }
     

common/djangoapps/student/views.py

@@ -8,7 +8,6 @@ import uuid
 import feedparser
 import urllib
 import itertools
-from collections import defaultdict
 
 from django.conf import settings
 from django.contrib.auth import logout, authenticate, login
@@ -37,6 +36,7 @@ from xmodule.modulestore.exceptions import ItemNotFoundError
 from models import Registration, UserProfile, PendingNameChange, PendingEmailChange, CourseEnrollment
 from datetime import date
 from collections import namedtuple
+from courseware.courses import course_staff_group_name, has_staff_access_to_course, get_courses_by_university
 
 log = logging.getLogger("mitx.student")
 Article = namedtuple('Article', 'title url author image deck publication publish_date')
@@ -64,9 +64,9 @@ def index(request):
         from external_auth.views import edXauth_ssl_login
         return edXauth_ssl_login(request)
 
-    return main_index()
+    return main_index(user=request.user)
 
-def main_index(extra_context = {}):
+def main_index(extra_context = {}, user=None):
     '''
     Render the edX main page.
 
@@ -88,11 +88,8 @@ def main_index(extra_context = {}):
         entry.image = soup.img['src'] if soup.img else None
         entry.summary = soup.getText()
 
-    universities = defaultdict(list)
-    courses = sorted(modulestore().get_courses(), key=lambda course: course.number)
-    for course in courses:
-        universities[course.org].append(course)
-
+    # The course selection work is done in courseware.courses.
+    universities = get_courses_by_university(None)
     context = {'universities': universities, 'entries': entries}
     context.update(extra_context)
     return render_to_response('index.html', context)
@@ -184,6 +181,14 @@ def change_enrollment(request):
                       .format(user.username, enrollment.course_id))
             return {'success': False, 'error': 'The course requested does not exist.'}
 
+        if settings.MITX_FEATURES.get('ACCESS_REQUIRE_STAFF_FOR_COURSE'):
+            # require that user be in the staff_* group (or be an overall admin) to be able to enroll
+            # eg staff_6.002x or staff_6.00x
+            if not has_staff_access_to_course(user,course):
+                staff_group = course_staff_group_name(course)
+                log.debug('user %s denied enrollment to %s ; not in %s' % (user,course.location.url(),staff_group))
+                return {'success': False, 'error' : '%s membership required to access course.' % staff_group}
+
         enrollment, created = CourseEnrollment.objects.get_or_create(user=user, course_id=course.id)
         return {'success': True}
 

common/djangoapps/track/models.py

@@ -1,3 +1,21 @@
 from django.db import models
 
-# Create your models here.
+from django.db import models
+
+class TrackingLog(models.Model):
+    dtcreated = models.DateTimeField('creation date',auto_now_add=True)
+    username = models.CharField(max_length=32,blank=True)
+    ip = models.CharField(max_length=32,blank=True)
+    event_source = models.CharField(max_length=32)
+    event_type = models.CharField(max_length=32,blank=True)
+    event = models.TextField(blank=True)
+    agent = models.CharField(max_length=256,blank=True)
+    page = models.CharField(max_length=32,blank=True,null=True)
+    time = models.DateTimeField('event time')
+
+    def __unicode__(self):
+        s = "[%s] %s@%s: %s | %s | %s | %s" % (self.time, self.username, self.ip, self.event_source,
+                                               self.event_type, self.page, self.event)
+        return s
+
+

common/djangoapps/track/views.py

@@ -2,19 +2,32 @@ import json
 import logging
 import os
 import datetime
+import dateutil.parser
 
-# Create your views here.
+from django.contrib.auth.decorators import login_required
 from django.http import HttpResponse
 from django.http import Http404
+from django.shortcuts import redirect
 from django.conf import settings
+from mitxmako.shortcuts import render_to_response
+
+from django_future.csrf import ensure_csrf_cookie
+from track.models import TrackingLog
 
 log = logging.getLogger("tracking")
 
+LOGFIELDS = ['username','ip','event_source','event_type','event','agent','page','time']
 
 def log_event(event):
     event_str = json.dumps(event)
     log.info(event_str[:settings.TRACK_MAX_EVENT])
-
+    if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'):
+        event['time'] = dateutil.parser.parse(event['time'])
+        tldat = TrackingLog(**dict( (x,event[x]) for x in LOGFIELDS ))
+        try:
+            tldat.save()
+        except Exception as err:
+            log.exception(err)
 
 def user_track(request):
     try:  # TODO: Do the same for many of the optional META parameters
@@ -70,4 +83,16 @@ def server_track(request, event_type, event, page=None):
         "page": page,
         "time": datetime.datetime.utcnow().isoformat(),
         }
+
+    if event_type=="/event_logs" and request.user.is_staff:	# don't log
+        return
     log_event(event)
+
+@login_required
+@ensure_csrf_cookie
+def view_tracking_log(request):
+    if not request.user.is_staff:
+        return redirect('/')
+    record_instances = TrackingLog.objects.all().order_by('-time')[0:100]
+    return render_to_response('tracking_log.html',{'records':record_instances})
+

common/djangoapps/xmodule_modifiers.py

@@ -1,9 +1,15 @@
+import re
 import json
+import logging
+
 from django.conf import settings
 from functools import wraps
 from static_replace import replace_urls
 from mitxmako.shortcuts import render_to_string
+from xmodule.seq_module import SequenceModule
+from xmodule.vertical_module import VerticalModule
 
+log = logging.getLogger("mitx.xmodule_modifiers")
 
 def wrap_xmodule(get_html, module, template):
     """
@@ -69,27 +75,31 @@ def add_histogram(get_html, module):
     the output of the old get_html function with additional information
     for admin users only, including a histogram of student answers and the
     definition of the xmodule
+
+    Does nothing if module is a SequenceModule
     """
     @wraps(get_html)
     def _get_html():
+
+        if type(module) in [SequenceModule, VerticalModule]:	# TODO: make this more general, eg use an XModule attribute instead
+            return get_html()
+
         module_id = module.id
         histogram = grade_histogram(module_id)
         render_histogram = len(histogram) > 0
 
-        # TODO: fixme - no filename in module.xml in general (this code block
-        # for edx4edx) the following if block is for summer 2012 edX course
-        # development; it will change when the CMS comes online
-        if settings.MITX_FEATURES.get('DISPLAY_EDIT_LINK') and settings.DEBUG and module_xml.get('filename') is not None:
-            coursename = multicourse_settings.get_coursename_from_request(request)
-            github_url = multicourse_settings.get_course_github_url(coursename)
-            fn = module_xml.get('filename')
-            if module_xml.tag=='problem': fn = 'problems/' + fn	# grrr
-            edit_link = (github_url + '/tree/master/' + fn) if github_url is not None else None
-            if module_xml.tag=='problem': edit_link += '.xml'	# grrr
+        # TODO (ichuang): Remove after fall 2012 LMS migration done
+        if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):
+            [filepath, filename] = module.definition.get('filename','')
+            osfs = module.system.filestore
+            if osfs.exists(filename):
+                filepath = filename	# if original, unmangled filename exists then use it (github doesn't like symlinks)
+            data_dir = osfs.root_path.rsplit('/')[-1]
+            edit_link = "https://github.com/MITx/%s/tree/master/%s" % (data_dir,filepath)
         else:
             edit_link = False
 
-        staff_context = {'definition': json.dumps(module.definition, indent=4),
+        staff_context = {'definition': module.definition.get('data'),
                          'metadata': json.dumps(module.metadata, indent=4),
                          'element_id': module.location.html_id(),
                          'edit_link': edit_link,
@@ -99,3 +109,4 @@ def add_histogram(get_html, module):
         return render_to_string("staff_problem_info.html", staff_context)
 
     return _get_html
+

common/lib/xmodule/xmodule/html_module.py

@@ -94,7 +94,15 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
                         msg = "Couldn't parse html in {0}.".format(filepath)
                         log.warning(msg)
                         system.error_tracker("Warning: " + msg)
-                    return {'data' : html}
+
+                    definition = {'data' : html}
+
+                    # TODO (ichuang): remove this after migration
+                    # for Fall 2012 LMS migration: keep filename (and unmangled filename)
+                    definition['filename'] = [ filepath, filename ]
+
+                    return definition
+
             except (ResourceNotFoundError) as err:
                 msg = 'Unable to load file contents at path {0}: {1} '.format(
                     filepath, err)

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -146,19 +146,30 @@ class XMLModuleStore(ModuleStoreBase):
                            os.path.exists(self.data_dir / d / "course.xml")]
 
         for course_dir in course_dirs:
-            try:
-                # Special-case code here, since we don't have a location for the
-                # course before it loads.
-                # So, make a tracker to track load-time errors, then put in the right
-                # place after the course loads and we have its location
-                errorlog = make_error_tracker()
-                course_descriptor = self.load_course(course_dir, errorlog.tracker)
-                self.courses[course_dir] = course_descriptor
-                self._location_errors[course_descriptor.location] = errorlog
-            except:
-                msg = "Failed to load course '%s'" % course_dir
-                log.exception(msg)
+            self.try_load_course(course_dir)
 
+    def try_load_course(self,course_dir):
+        '''
+        Load a course, keeping track of errors as we go along.
+        '''
+        try:
+            # Special-case code here, since we don't have a location for the
+            # course before it loads.
+            # So, make a tracker to track load-time errors, then put in the right
+            # place after the course loads and we have its location
+            errorlog = make_error_tracker()
+            course_descriptor = self.load_course(course_dir, errorlog.tracker)
+            self.courses[course_dir] = course_descriptor
+            self._location_errors[course_descriptor.location] = errorlog
+        except:
+            msg = "Failed to load course '%s'" % course_dir
+            log.exception(msg)
+
+    def __unicode__(self):
+        '''
+        String representation - for debugging
+        '''
+        return '<XMLModuleStore>data_dir=%s, %d courses, %d modules' % (self.data_dir,len(self.courses),len(self.modules))
 
     def load_course(self, course_dir, tracker):
         """

common/lib/xmodule/xmodule/x_module.py

@@ -204,6 +204,8 @@ class XModule(HTMLSnippet):
         '''
         return self.metadata.get('display_name',
                                  self.url_name.replace('_', ' '))
+    def __unicode__(self):
+        return '<x_module(name=%s, category=%s, id=%s)>' % (self.name, self.category, self.id)
 
     def get_children(self):
         '''

common/lib/xmodule/xmodule/xml_module.py

@@ -41,6 +41,7 @@ class XmlDescriptor(XModuleDescriptor):
     # to definition_from_xml, and from the xml returned by definition_to_xml
     metadata_attributes = ('format', 'graceperiod', 'showanswer', 'rerandomize',
         'start', 'due', 'graded', 'display_name', 'url_name', 'hide_from_toc',
+        'ispublic', 	# if True, then course is listed for all users; see 
         # VS[compat] Remove once unused.
         'name', 'slug')
 
@@ -109,6 +110,7 @@ class XmlDescriptor(XModuleDescriptor):
         filename = xml_object.get('filename')
         if filename is None:
             definition_xml = copy.deepcopy(xml_object)
+            filepath = ''
         else:
             filepath = cls._format_filepath(xml_object.tag, filename)
 
@@ -136,7 +138,13 @@ class XmlDescriptor(XModuleDescriptor):
                 raise Exception, msg, sys.exc_info()[2]
 
         cls.clean_metadata_from_xml(definition_xml)
-        return cls.definition_from_xml(definition_xml, system)
+        definition = cls.definition_from_xml(definition_xml, system)
+
+        # TODO (ichuang): remove this after migration
+        # for Fall 2012 LMS migration: keep filename (and unmangled filename)
+        definition['filename'] = [ filepath, filename ]
+
+        return definition
 
 
     @classmethod

lms/djangoapps/courseware/courses.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from fs.errors import ResourceNotFoundError
 from functools import wraps
 import logging
@@ -114,3 +115,57 @@ def get_course_info_section(course, section_key):
             return "! Info section missing !"
 
     raise KeyError("Invalid about key " + str(section_key))
+
+def course_staff_group_name(course):
+    '''
+    course should be either a CourseDescriptor instance, or a string (the .course entry of a Location)
+    '''
+    if isinstance(course,str):
+        coursename = course
+    else:
+        coursename = course.metadata.get('data_dir','UnknownCourseName')
+    if not coursename:					# Fall 2012: not all course.xml have metadata correct yet
+        coursename = course.metadata.get('course','')
+    return 'staff_%s' % coursename
+
+def has_staff_access_to_course(user,course):
+    '''
+    Returns True if the given user has staff access to the course.
+    This means that user is in the staff_* group, or is an overall admin.
+    '''
+    if user is None or (not user.is_authenticated()) or course is None:
+        return False
+    if user.is_staff:
+        return True
+    user_groups = [x[1] for x in user.groups.values_list()]	# note this is the Auth group, not UserTestGroup
+    staff_group = course_staff_group_name(course)
+    log.debug('course %s user %s groups %s' % (staff_group, user, user_groups))
+    if staff_group in user_groups:
+        return True
+    return False
+
+def has_access_to_course(user,course):
+    if course.metadata.get('ispublic'):
+        return True
+    return has_staff_access_to_course(user,course)
+
+def get_courses_by_university(user):
+    '''
+    Returns dict of lists of courses available, keyed by course.org (ie university).
+    Courses are sorted by course.number.
+
+    if ACCESS_REQUIRE_STAFF_FOR_COURSE then list only includes those accessible to user.
+    '''
+    # TODO: Clean up how 'error' is done.
+    # filter out any courses that errored.
+    courses = [c for c in modulestore().get_courses()
+               if isinstance(c, CourseDescriptor)]
+    courses = sorted(courses, key=lambda course: course.number)
+    universities = defaultdict(list)
+    for course in courses:
+        if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):
+            if not has_access_to_course(user,course):
+                continue
+        universities[course.org].append(course)
+    return universities
+    

lms/djangoapps/courseware/module_render.py

@@ -15,6 +15,8 @@ from xmodule.exceptions import NotFoundError
 from xmodule.x_module import ModuleSystem
 from xmodule_modifiers import replace_static_urls, add_histogram, wrap_xmodule
 
+from courseware.courses import has_staff_access_to_course
+
 log = logging.getLogger("mitx.courseware")
 
 
@@ -188,8 +190,9 @@ def get_module(user, request, location, student_module_cache, position=None):
         module.metadata['data_dir']
     )
 
-    if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF') and user.is_staff:
-        module.get_html = add_histogram(module.get_html, module)
+    if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF'):
+        if has_staff_access_to_course(user, module.location.course):
+            module.get_html = add_histogram(module.get_html, module)
 
     # If StudentModule for this instance wasn't already in the database,
     # and this isn't a guest user, create it.

lms/djangoapps/courseware/views.py

@@ -1,4 +1,3 @@
-from collections import defaultdict
 import json
 import logging
 import urllib
@@ -28,7 +27,7 @@ from xmodule.course_module import CourseDescriptor
 from util.cache import cache, cache_if_anonymous
 from student.models import UserTestGroup, CourseEnrollment
 from courseware import grades
-from courseware.courses import check_course
+from courseware.courses import check_course, get_courses_by_university
 
 
 log = logging.getLogger("mitx.courseware")
@@ -58,19 +57,12 @@ def user_groups(user):
 @ensure_csrf_cookie
 @cache_if_anonymous
 def courses(request):
-    # TODO: Clean up how 'error' is done.
-
-    # filter out any courses that errored.
-    courses = [c for c in modulestore().get_courses()
-               if isinstance(c, CourseDescriptor)]
-    courses = sorted(courses, key=lambda course: course.number)
-    universities = defaultdict(list)
-    for course in courses:
-        universities[course.org].append(course)
-
+    '''
+    Render "find courses" page.  The course selection work is done in courseware.courses.
+    '''
+    universities = get_courses_by_university(request.user)
     return render_to_response("courses.html", {'universities': universities})
 
-
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
 def gradebook(request, course_id):
     if 'course_admin' not in user_groups(request.user):
@@ -150,6 +142,7 @@ def render_accordion(request, course, chapter, section):
     return render_to_string('accordion.html', context)
 
 
+@login_required
 @ensure_csrf_cookie
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
 def index(request, course_id, chapter=None, section=None,
@@ -172,6 +165,10 @@ def index(request, course_id, chapter=None, section=None,
      - HTTPresponse
     '''
     course = check_course(course_id)
+    registered = registered_for_course(course, request.user)
+    if not registered:
+        log.debug('User %s tried to view course %s but is not enrolled' % (request.user,course.location.url()))
+        return redirect(reverse('about_course', args=[course.id]))
 
     try:
         context = {
@@ -266,14 +263,18 @@ def course_info(request, course_id):
     return render_to_response('info.html', {'course': course})
 
 
+def registered_for_course(course, user):
+    '''Return CourseEnrollment if user is registered for course, else False'''
+    if user is None:
+        return False
+    if user.is_authenticated():
+        return CourseEnrollment.objects.filter(user=user, course_id=course.id).exists()
+    else:
+        return False
+
 @ensure_csrf_cookie
 @cache_if_anonymous
 def course_about(request, course_id):
-    def registered_for_course(course, user):
-        if user.is_authenticated():
-            return CourseEnrollment.objects.filter(user=user, course_id=course.id).exists()
-        else:
-            return False
     course = check_course(course_id, course_must_be_open=False)
     registered = registered_for_course(course, request.user)
     return render_to_response('portal/course_about.html', {'course': course, 'registered': registered})
@@ -288,7 +289,7 @@ def university_profile(request, org_id):
         raise Http404("University Profile not found for {0}".format(org_id))
 
     # Only grab courses for this org...
-    courses = [c for c in all_courses if c.org == org_id]
+    courses = get_courses_by_university(request.user)[org_id]
     context = dict(courses=courses, org_id=org_id)
     template_file = "university_profile/{0}.html".format(org_id).lower()
 

lms/djangoapps/lms_migration/migrate.py

@@ -0,0 +1,110 @@
+#
+# migration tools for content team to go from stable-edx4edx to LMS+CMS
+#
+
+import logging
+from pprint import pprint
+import xmodule.modulestore.django as xmodule_django
+from xmodule.modulestore.django import modulestore
+
+from django.http import HttpResponse
+from django.conf import settings
+
+log = logging.getLogger("mitx.lms_migrate")
+LOCAL_DEBUG = True
+ALLOWED_IPS = settings.LMS_MIGRATION_ALLOWED_IPS
+
+def escape(s):
+    """escape HTML special characters in string"""
+    return str(s).replace('<','&lt;').replace('>','&gt;')
+
+def manage_modulestores(request,reload_dir=None):
+    '''
+    Manage the static in-memory modulestores.
+
+    If reload_dir is not None, then instruct the xml loader to reload that course directory.
+    '''
+    html = "<html><body>"
+
+    def_ms = modulestore()
+    courses = def_ms.get_courses()
+
+    #----------------------------------------
+    # check on IP address of requester
+
+    ip = request.META.get('HTTP_X_REAL_IP','')	# nginx reverse proxy
+    if not ip:
+        ip = request.META.get('REMOTE_ADDR','None')
+    
+    if LOCAL_DEBUG:
+        html += '<h3>IP address: %s ' % ip
+        html += '<h3>User: %s ' % request.user
+        log.debug('request from ip=%s, user=%s' % (ip,request.user))
+
+    if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
+        if request.user and request.user.is_staff:
+            log.debug('request allowed because user=%s is staff' % request.user)
+        else:
+            html += 'Permission denied'
+            html += "</body></html>"
+            log.debug('request denied, ALLOWED_IPS=%s' % ALLOWED_IPS)
+            return HttpResponse(html)    
+
+    #----------------------------------------
+    # reload course if specified
+
+    if reload_dir is not None:
+        if reload_dir not in def_ms.courses:
+            html += "<h2><font color='red'>Error: '%s' is not a valid course directory</font></h2>" % reload_dir
+        else:
+            html += "<h2><font color='blue'>Reloaded course directory '%s'</font></h2>" % reload_dir
+            def_ms.try_load_course(reload_dir)
+
+    #----------------------------------------
+
+    html += '<h2>Courses loaded in the modulestore</h2>'
+    html += '<ol>'
+    for cdir, course in def_ms.courses.items():
+        html += '<li><a href="%s/migrate/reload/%s">%s</a> (%s)</li>' % (settings.MITX_ROOT_URL,
+                                                            escape(cdir),
+                                                            escape(cdir),
+                                                            course.location.url())
+    html += '</ol>'
+
+    #----------------------------------------
+
+    dumpfields = ['definition','location','metadata']
+    
+    for cdir, course in def_ms.courses.items():
+        html += '<hr width="100%"/>'
+        html += '<h2>Course: %s (%s)</h2>' % (course.metadata['display_name'],cdir)
+
+        for field in dumpfields:
+            data = getattr(course,field)
+            html += '<h3>%s</h3>' % field
+            if type(data)==dict:
+                html += '<ul>'
+                for k,v in data.items():
+                    html += '<li>%s:%s</li>' % (escape(k),escape(v))
+                html += '</ul>'
+            else:
+                html += '<ul><li>%s</li></ul>' % escape(data)
+        
+
+    #----------------------------------------
+
+    html += '<hr width="100%"/>'
+    html += "courses: <pre>%s</pre>" % escape(courses)
+
+    ms = xmodule_django._MODULESTORES
+    html += "modules: <pre>%s</pre>" % escape(ms)
+    html += "default modulestore: <pre>%s</pre>" % escape(unicode(def_ms))
+
+    #----------------------------------------
+
+    log.debug('_MODULESTORES=%s' % ms)
+    log.debug('courses=%s' % courses)
+    log.debug('def_ms=%s' % unicode(def_ms))
+
+    html += "</body></html>"
+    return HttpResponse(html)    

lms/envs/common.py

@@ -48,6 +48,17 @@ MITX_FEATURES = {
     ## DO NOT SET TO True IN THIS FILE
     ## Doing so will cause all courses to be released on production
     'DISABLE_START_DATES': False,  # When True, all courses will be active, regardless of start date
+
+    'ENABLE_TEXTBOOK' : True,
+    'ENABLE_DISCUSSION' : True,
+
+    'ENABLE_SQL_TRACKING_LOGS': False,
+    'ENABLE_LMS_MIGRATION': False,
+
+    # extrernal access methods
+    'ACCESS_REQUIRE_STAFF_FOR_COURSE': False,
+    'AUTH_USE_OPENID': False,
+    'AUTH_USE_MIT_CERTIFICATES' : False,
 }
 
 # Used for A/B testing

lms/envs/dev.py

@@ -14,6 +14,7 @@ DEBUG = True
 TEMPLATE_DEBUG = True
 
 MITX_FEATURES['DISABLE_START_DATES'] = True
+MITX_FEATURES['ENABLE_SQL_TRACKING_LOGS'] = True
 
 WIKI_ENABLED = True
 
@@ -58,6 +59,12 @@ CACHE_TIMEOUT = 0
 # Dummy secret key for dev
 SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'
 
+################################ LMS Migration #################################
+MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True
+MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = False   # require that user be in the staff_* group to be able to enroll
+
+LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1']
+
 ################################ OpenID Auth #################################
 MITX_FEATURES['AUTH_USE_OPENID'] = True
 MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True

lms/envs/dev_ike.py

@@ -9,108 +9,10 @@ sessions. Assumes structure:
 """
 from .common import *
 from .logsettings import get_logger_config
+from .dev import *
 
-DEBUG = True
-TEMPLATE_DEBUG = True
+WIKI_ENABLED = False
+MITX_FEATURES['ENABLE_TEXTBOOK'] = False
+MITX_FEATURES['ENABLE_DISCUSSION'] = False
+MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True	  # require that user be in the staff_* group to be able to enroll
 
-MITX_FEATURES['DISABLE_START_DATES'] = True
-
-WIKI_ENABLED = True
-
-LOGGING = get_logger_config(ENV_ROOT / "log", 
-                            logging_env="dev",
-                            tracking_filename="tracking.log",
-                            debug=True)
-
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': ENV_ROOT / "db" / "mitx.db",
-    }
-}
-
-CACHES = {
-    # This is the cache used for most things. Askbot will not work without a 
-    # functioning cache -- it relies on caching to load its settings in places.
-    # In staging/prod envs, the sessions also live here.
-    'default': {
-        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
-        'LOCATION': 'mitx_loc_mem_cache',
-        'KEY_FUNCTION': 'util.memcache.safe_key',
-    },
-
-    # The general cache is what you get if you use our util.cache. It's used for
-    # things like caching the course.xml file for different A/B test groups.
-    # We set it to be a DummyCache to force reloading of course.xml in dev.
-    # In staging environments, we would grab VERSION from data uploaded by the
-    # push process.
-    'general': {
-        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
-        'KEY_PREFIX': 'general',
-        'VERSION': 4,
-        'KEY_FUNCTION': 'util.memcache.safe_key',
-    }
-}
-
-# Dummy secret key for dev
-SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'
-
-################################ OpenID Auth #################################
-MITX_FEATURES['AUTH_USE_OPENID'] = True
-
-INSTALLED_APPS += ('external_auth',) 
-INSTALLED_APPS += ('django_openid_auth',)
-#INSTALLED_APPS += ('ssl_auth',)
-
-#MIDDLEWARE_CLASSES += (
-#    #'ssl_auth.ssl_auth.NginxProxyHeaderMiddleware',		# ssl authentication behind nginx proxy
-#    )
-
-#AUTHENTICATION_BACKENDS = (
-#    'django_openid_auth.auth.OpenIDBackend',
-#    'django.contrib.auth.backends.ModelBackend',
-#    )
-
-OPENID_CREATE_USERS = False
-OPENID_UPDATE_DETAILS_FROM_SREG = True
-OPENID_SSO_SERVER_URL = 'https://www.google.com/accounts/o8/id'
-OPENID_USE_AS_ADMIN_LOGIN = False
-#import external_auth.views as edXauth
-#OPENID_RENDER_FAILURE = edXauth.edXauth_openid
-
-################################ DEBUG TOOLBAR #################################
-INSTALLED_APPS += ('debug_toolbar',) 
-MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',)
-INTERNAL_IPS = ('127.0.0.1',)
-
-DEBUG_TOOLBAR_PANELS = (
-   'debug_toolbar.panels.version.VersionDebugPanel',
-   'debug_toolbar.panels.timer.TimerDebugPanel',
-   'debug_toolbar.panels.settings_vars.SettingsVarsDebugPanel',
-   'debug_toolbar.panels.headers.HeaderDebugPanel',
-   'debug_toolbar.panels.request_vars.RequestVarsDebugPanel',
-   'debug_toolbar.panels.sql.SQLDebugPanel',
-   'debug_toolbar.panels.signals.SignalDebugPanel',
-   'debug_toolbar.panels.logger.LoggingPanel',
-
-#  Enabling the profiler has a weird bug as of django-debug-toolbar==0.9.4 and
-#  Django=1.3.1/1.4 where requests to views get duplicated (your method gets 
-#  hit twice). So you can uncomment when you need to diagnose performance 
-#  problems, but you shouldn't leave it on.
-#  'debug_toolbar.panels.profiling.ProfilingDebugPanel',
-)
-
-############################ FILE UPLOADS (ASKBOT) #############################
-DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
-MEDIA_ROOT = ENV_ROOT / "uploads"
-MEDIA_URL = "/static/uploads/"
-STATICFILES_DIRS.append(("uploads", MEDIA_ROOT))
-FILE_UPLOAD_TEMP_DIR = ENV_ROOT / "uploads"
-FILE_UPLOAD_HANDLERS = (
-    'django.core.files.uploadhandler.MemoryFileUploadHandler',
-    'django.core.files.uploadhandler.TemporaryFileUploadHandler',
-)
-
-########################### PIPELINE #################################
-
-PIPELINE_SASS_ARGUMENTS = '-r {proj_dir}/static/sass/bourbon/lib/bourbon.rb'.format(proj_dir=PROJECT_ROOT)

lms/templates/course_navigation.html

@@ -14,10 +14,16 @@ def url_class(url):
       <li class="courseware"><a href="${reverse('courseware', args=[course.id])}" class="${url_class('courseware')}">Courseware</a></li>
       <li class="info"><a href="${reverse('info', args=[course.id])}" class="${url_class('info')}">Course Info</a></li>
 % if user.is_authenticated():
+% if settings.MITX_FEATURES.get('ENABLE_TEXTBOOK'):
       <li class="book"><a href="${reverse('book', args=[course.id])}" class="${url_class('book')}">Textbook</a></li>
+% endif 
+% if settings.MITX_FEATURES.get('ENABLE_DISCUSSION'):
       <li class="discussion"><a href="${reverse('questions')}">Discussion</a></li>
+% endif 
 % endif
+% if settings.WIKI_ENABLED:
       <li class="wiki"><a href="${reverse('wiki_root', args=[course.id])}" class="${url_class('wiki')}">Wiki</a></li>
+% endif
 % if user.is_authenticated():
       <li class="profile"><a href="${reverse('profile', args=[course.id])}" class="${url_class('profile')}">Profile</a></li>
 % endif

lms/templates/portal/course_about.html

@@ -19,6 +19,8 @@
         $(document).delegate('#class_enroll_form', 'ajax:success', function(data, json, xhr) {
           if(json.success) {
             location.href="${reverse('dashboard')}";
+          }else{
+	    $('#register_message).html("<p><font color='red'>" + json.error + "</font></p>")
           }
         });
       })(this)
@@ -60,9 +62,24 @@
         <div class="main-cta">
           %if user.is_authenticated():
             %if registered:
+   	        <%
+		  ## TODO: move this logic into a view
+                  if course.has_started() or settings.MITX_FEATURES['DISABLE_START_DATES']:
+                      course_target = reverse('info', args=[course.id])
+                  else:
+                      course_target = reverse('about_course', args=[course.id])
+                  show_link = settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION')
+                %>
+	        %if show_link:
+                <a href="${course_target}">
+                %endif
                 <span class="register disabled">You are registered for this course (${course.number}).</span>
+	        %if show_link:
+                </a>
+                %endif
             %else:
                 <a href="#" class="register">Register for ${course.number}</a>
+	        <div id="register_message"></div>
             %endif
           %else:
             <a href="#signup-modal" class="register" rel="leanModal" data-notice='You must Sign Up or <a href="#login-modal" rel="leanModal">Log In</a> to enroll.'>Register for ${course.number}</a>

lms/templates/staff_problem_info.html

@@ -1,11 +1,11 @@
 ${module_content}
-<div class="staff_info">
-definition = ${definition | h}
-metadata = ${metadata | h}
-</div>
 %if edit_link:
 <div><a href="${edit_link}">Edit</a></div>
 % endif
+<div class="staff_info">
+definition = <pre>${definition | h}</pre>
+metadata = ${metadata | h}
+</div>
 %if render_histogram:
 <div id="histogram_${element_id}" class="histogram" data-histogram="${histogram}"></div>
 %endif

lms/templates/tracking_log.html

@@ -0,0 +1,14 @@
+<html>
+<h1>Tracking Log</h1>
+<table border="1"><tr><th>datetime</th><th>username</th><th>ipaddr</th><th>source</th><th>type</th></tr>
+% for rec in records:
+    <tr>
+    <td>${rec.time}</td> 
+    <td>${rec.username}</td>  
+    <td>${rec.ip}</td>  
+    <td>${rec.event_source}</td>   
+    <td>${rec.event_type}</td>   
+    </tr>
+% endfor
+</table>    
+</html>

lms/urls.py

@@ -169,6 +169,17 @@ if settings.MITX_FEATURES.get('AUTH_USE_OPENID'):
         url(r'^openid/logo.gif$', 'django_openid_auth.views.logo', name='openid-logo'),
         )
 
+if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'):
+    urlpatterns += (
+        url(r'^migrate/modules$', 'lms_migration.migrate.manage_modulestores'),
+        url(r'^migrate/reload/(?P<reload_dir>[^/]+)$', 'lms_migration.migrate.manage_modulestores'),
+        )
+
+if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'):
+    urlpatterns += (
+        url(r'^event_logs$', 'track.views.view_tracking_log'),
+        )
+
 urlpatterns = patterns(*urlpatterns)
 
 if settings.DEBUG:

utility-scripts/create_groups.py

@@ -0,0 +1,35 @@
+#!/usr/bin/python
+#
+# File:   create_groups.py
+#
+# Create all staff_* groups for classes in data directory.
+
+import os, sys, string, re
+
+sys.path.append(os.path.abspath('.'))
+os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev'
+
+try:
+    from lms.envs.dev import *
+except Exception as err:
+    print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory."
+    sys.exit(-1)
+
+from django.conf import settings
+from django.contrib.auth.models import User, Group
+from path import path
+
+data_dir = settings.DATA_DIR
+print "data_dir = %s" % data_dir
+
+for course_dir in os.listdir(data_dir):
+    # print course_dir
+    if not os.path.isdir(path(data_dir) / course_dir):
+        continue
+    gname = 'staff_%s' % course_dir
+    if Group.objects.filter(name=gname):
+        print "group exists for %s" % gname
+        continue
+    g = Group(name=gname)
+    g.save()
+    print "created group %s" % gname

utility-scripts/create_user.py

@@ -0,0 +1,149 @@
+#!/usr/bin/python
+#
+# File:   create_user.py
+#
+# Create user.  Prompt for groups and ExternalAuthMap
+
+import os, sys, string, re
+import datetime
+from getpass import getpass
+import json
+import readline
+
+sys.path.append(os.path.abspath('.'))
+os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev'
+
+try:
+    from lms.envs.dev import *
+except Exception as err:
+    print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory."
+    sys.exit(-1)
+
+from student.models import UserProfile, Registration
+from external_auth.models import ExternalAuthMap
+from django.contrib.auth.models import User, Group
+from random import choice
+
+class MyCompleter(object):  # Custom completer
+
+    def __init__(self, options):
+        self.options = sorted(options)
+
+    def complete(self, text, state):
+        if state == 0:  # on first trigger, build possible matches
+            if text:  # cache matches (entries that start with entered text)
+                self.matches = [s for s in self.options 
+                                    if s and s.startswith(text)]
+            else:  # no text entered, all matches possible
+                self.matches = self.options[:]
+
+        # return match indexed by state
+        try: 
+            return self.matches[state]
+        except IndexError:
+            return None
+
+def GenPasswd(length=8, chars=string.letters + string.digits):
+    return ''.join([choice(chars) for i in range(length)])
+
+#-----------------------------------------------------------------------------
+# main
+
+while True:
+    uname = raw_input('username: ')
+    if User.objects.filter(username=uname):
+        print "username %s already taken" % uname
+    else:
+        break
+
+make_eamap = False
+if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y':
+    email = '%s@MIT.EDU' % uname
+    if not email.endswith('@MIT.EDU'):
+        print "Failed - email must be @MIT.EDU"
+        sys.exit(-1)
+    mit_domain = 'ssl:MIT'
+    if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain):
+        print "Failed - email %s already exists as external_id" % email
+        sys.exit(-1)
+    make_eamap = True
+    password = GenPasswd(12)
+    
+    # get name from kerberos
+    kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip()
+    name = raw_input('Full name: [%s] ' % kname).strip()
+    if name=='':
+        name = kname
+    print "name = %s" % name
+else:
+    while True:
+        password = getpass()
+        password2 = getpass()
+        if password == password2:
+            break
+        print "Oops, passwords do not match, please retry"
+
+    while True:
+        email = raw_input('email: ')
+        if User.objects.filter(email=email):
+            print "email %s already taken" % email
+        else:
+            break
+        
+    name = raw_input('Full name: ')
+
+        
+user = User(username=uname, email=email, is_active=True)
+user.set_password(password)
+try:
+    user.save()
+except IntegrityError:
+    print "Oops, failed to create user %s, IntegrityError" % user
+    raise
+    
+r = Registration()
+r.register(user)
+
+up = UserProfile(user=user)
+up.name = name
+up.save()
+
+if make_eamap:
+    credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email)
+    eamap = ExternalAuthMap(external_id = email,
+                            external_email = email,
+                            external_domain = mit_domain,
+                            external_name = name,
+                            internal_password = password,
+                            external_credentials = json.dumps(credentials),
+        )
+    eamap.user = user
+    eamap.dtsignup = datetime.datetime.now()
+    eamap.save()
+
+print "User %s created successfully!" % user
+
+if not raw_input('Add user %s to any groups? [n] ' % user).lower()=='y':
+    sys.exit(0)
+
+print "Here are the groups available:"
+
+groups = [str(g.name) for g in Group.objects.all()]
+print groups
+
+completer = MyCompleter(groups)
+readline.set_completer(completer.complete)
+readline.parse_and_bind('tab: complete')
+
+while True:
+    gname = raw_input("Add group (tab to autocomplete, empty line to end): ")
+    if not gname:
+        break
+    if not gname in groups:
+        print "Unknown group %s" % gname
+        continue
+    g = Group.objects.get(name=gname)
+    user.groups.add(g)
+    print "Added %s to group %s" % (user,g)
+
+print "Done!"