diff --git a/Gemfile b/Gemfile index c6a19caca2..9ad08c7adb 100644 --- a/Gemfile +++ b/Gemfile @@ -1,5 +1,5 @@ source :rubygems - +ruby "1.9.3" gem 'rake' gem 'sass', '3.1.15' gem 'bourbon', '~> 1.3.6' diff --git a/cms/djangoapps/contentstore/management/commands/import.py b/cms/djangoapps/contentstore/management/commands/import.py index 69aaa35a7d..a6790ad59b 100644 --- a/cms/djangoapps/contentstore/management/commands/import.py +++ b/cms/djangoapps/contentstore/management/commands/import.py @@ -23,4 +23,7 @@ class Command(BaseCommand): course_dirs = args[1:] else: course_dirs = None + print "Importing. Data_dir={data}, course_dirs={courses}".format( + data=data_dir, + courses=course_dirs) import_from_xml(modulestore(), data_dir, course_dirs) diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index 6d5b2117c4..0305795e52 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -108,7 +108,7 @@ def edit_item(request): 'contents': item.get_html(), 'js_module': item.js_module_name, 'category': item.category, - 'name': item.name, + 'url_name': item.url_name, 'previews': get_module_previews(request, item), }) @@ -176,7 +176,7 @@ def load_preview_state(request, preview_id, location): def save_preview_state(request, preview_id, location, instance_state, shared_state): """ - Load the state of a preview module to the request + Save the state of a preview module to the request preview_id (str): An identifier specifying which preview this module is used for location: The Location of the module to dispatch to @@ -214,7 +214,10 @@ def preview_module_system(request, preview_id, descriptor): get_module=partial(get_preview_module, request, preview_id), render_template=render_from_lms, debug=True, - replace_urls=replace_urls + replace_urls=replace_urls, + # TODO (vshnayder): All CMS users get staff view by default + # is that what we want? + is_staff=True, ) diff --git a/cms/djangoapps/github_sync/tests/test_views.py b/cms/djangoapps/github_sync/tests/test_views.py index 212d707340..37030d6a1b 100644 --- a/cms/djangoapps/github_sync/tests/test_views.py +++ b/cms/djangoapps/github_sync/tests/test_views.py @@ -11,33 +11,33 @@ class PostReceiveTestCase(TestCase): def setUp(self): self.client = Client() - @patch('github_sync.views.sync_with_github') - def test_non_branch(self, sync_with_github): + @patch('github_sync.views.import_from_github') + def test_non_branch(self, import_from_github): self.client.post('/github_service_hook', {'payload': json.dumps({ 'ref': 'refs/tags/foo'}) }) - self.assertFalse(sync_with_github.called) + self.assertFalse(import_from_github.called) - @patch('github_sync.views.sync_with_github') - def test_non_watched_repo(self, sync_with_github): + @patch('github_sync.views.import_from_github') + def test_non_watched_repo(self, import_from_github): self.client.post('/github_service_hook', {'payload': json.dumps({ 'ref': 'refs/heads/branch', 'repository': {'name': 'bad_repo'}}) }) - self.assertFalse(sync_with_github.called) + self.assertFalse(import_from_github.called) - @patch('github_sync.views.sync_with_github') - def test_non_tracked_branch(self, sync_with_github): + @patch('github_sync.views.import_from_github') + def test_non_tracked_branch(self, import_from_github): self.client.post('/github_service_hook', {'payload': json.dumps({ 'ref': 'refs/heads/non_branch', 'repository': {'name': 'repo'}}) }) - self.assertFalse(sync_with_github.called) + self.assertFalse(import_from_github.called) - @patch('github_sync.views.sync_with_github') - def test_tracked_branch(self, sync_with_github): + @patch('github_sync.views.import_from_github') + def test_tracked_branch(self, import_from_github): self.client.post('/github_service_hook', {'payload': json.dumps({ 'ref': 'refs/heads/branch', 'repository': {'name': 'repo'}}) }) - sync_with_github.assert_called_with(load_repo_settings('repo')) + import_from_github.assert_called_with(load_repo_settings('repo')) diff --git a/cms/djangoapps/github_sync/views.py b/cms/djangoapps/github_sync/views.py index 941d50f986..c3b5172b29 100644 --- a/cms/djangoapps/github_sync/views.py +++ b/cms/djangoapps/github_sync/views.py @@ -5,7 +5,7 @@ from django.http import HttpResponse from django.conf import settings from django_future.csrf import csrf_exempt -from . import sync_with_github, load_repo_settings +from . import import_from_github, load_repo_settings log = logging.getLogger() @@ -46,6 +46,6 @@ def github_post_receive(request): log.info('Ignoring changes to non-tracked branch %s in repo %s' % (branch_name, repo_name)) return HttpResponse('Ignoring non-tracked branch') - sync_with_github(repo) + import_from_github(repo) return HttpResponse('Push received') diff --git a/cms/envs/dev.py b/cms/envs/dev.py index b0729ba885..c5e1cf5689 100644 --- a/cms/envs/dev.py +++ b/cms/envs/dev.py @@ -2,13 +2,18 @@ This config file runs the simplest dev environment""" from .common import * +from .logsettings import get_logger_config import logging import sys -logging.basicConfig(stream=sys.stdout, ) DEBUG = True TEMPLATE_DEBUG = DEBUG +LOGGING = get_logger_config(ENV_ROOT / "log", + logging_env="dev", + tracking_filename="tracking.log", + debug=True) + MODULESTORE = { 'default': { @@ -37,7 +42,8 @@ REPOS = { }, 'content-mit-6002x': { 'branch': 'master', - 'origin': 'git@github.com:MITx/6002x-fall-2012.git', + #'origin': 'git@github.com:MITx/6002x-fall-2012.git', + 'origin': 'git@github.com:MITx/content-mit-6002x.git', }, '6.00x': { 'branch': 'master', @@ -75,3 +81,6 @@ CACHES = { 'KEY_FUNCTION': 'util.memcache.safe_key', } } + +# Make the keyedcache startup warnings go away +CACHE_TIMEOUT = 0 diff --git a/cms/envs/logsettings.py b/cms/envs/logsettings.py index 31130e33c6..3683314d02 100644 --- a/cms/envs/logsettings.py +++ b/cms/envs/logsettings.py @@ -3,19 +3,19 @@ import os.path import platform import sys -def get_logger_config(log_dir, - logging_env="no_env", +def get_logger_config(log_dir, + logging_env="no_env", tracking_filename=None, syslog_addr=None, debug=False): """Return the appropriate logging config dictionary. You should assign the - result of this to the LOGGING var in your settings. The reason it's done + result of this to the LOGGING var in your settings. The reason it's done this way instead of registering directly is because I didn't want to worry - about resetting the logging state if this is called multiple times when + about resetting the logging state if this is called multiple times when settings are extended.""" # If we're given an explicit place to put tracking logs, we do that (say for - # debugging). However, logging is not safe for multiple processes hitting + # debugging). However, logging is not safe for multiple processes hitting # the same file. So if it's left blank, we dynamically create the filename # based on the PID of this worker process. if tracking_filename: @@ -33,6 +33,7 @@ def get_logger_config(log_dir, return { 'version': 1, + 'disable_existing_loggers': False, 'formatters' : { 'standard' : { 'format' : '%(asctime)s %(levelname)s %(process)d [%(name)s] %(filename)s:%(lineno)d - %(message)s', diff --git a/cms/static/sass/_base.scss b/cms/static/sass/_base.scss index cad315f6e4..2ea98473d1 100644 --- a/cms/static/sass/_base.scss +++ b/cms/static/sass/_base.scss @@ -2,6 +2,7 @@ $fg-column: 70px; $fg-gutter: 26px; $fg-max-columns: 12; $body-font-family: "Open Sans", "Lucida Grande", "Lucida Sans Unicode", "Lucida Sans", Geneva, Verdana, sans-serif; +$sans-serif: "Open Sans", "Lucida Grande", "Lucida Sans Unicode", "Lucida Sans", Geneva, Verdana, sans-serif; $body-font-size: 14px; $body-line-height: 20px; @@ -12,6 +13,7 @@ $orange: #f96e5b; $yellow: #fff8af; $cream: #F6EFD4; $mit-red: #933; +$border-color: #ddd; @mixin hide-text { background-color: transparent; diff --git a/cms/static/sass/_content-types.scss b/cms/static/sass/_content-types.scss index 00af06d5ad..e85d2a5c24 100644 --- a/cms/static/sass/_content-types.scss +++ b/cms/static/sass/_content-types.scss @@ -56,10 +56,10 @@ .module a:first-child { @extend .content-type; - background-image: url('/static/img/content-types/module.png'); + background-image: url('../img/content-types/module.png'); } .module a:first-child { @extend .content-type; - background-image: url('/static/img/content-types/module.png'); + background-image: url('../img/content-types/module.png'); } diff --git a/cms/templates/unit.html b/cms/templates/unit.html index 6aa780d42a..828f95ed47 100644 --- a/cms/templates/unit.html +++ b/cms/templates/unit.html @@ -1,7 +1,7 @@
-

${name}

+

${url_name}

${category}

diff --git a/cms/templates/widgets/navigation.html b/cms/templates/widgets/navigation.html index 9f9b37d571..ce18e867bd 100644 --- a/cms/templates/widgets/navigation.html +++ b/cms/templates/widgets/navigation.html @@ -41,7 +41,7 @@ % for week in weeks:
  • -

    ${week.name}

    +

    ${week.url_name}

      % if 'goals' in week.metadata: % for goal in week.metadata['goals']: @@ -60,7 +60,7 @@ data-type="${module.js_module_name}" data-preview-type="${module.module_class.js_module_name}"> - ${module.name} + ${module.url_name} handle % endfor diff --git a/cms/templates/widgets/sequence-edit.html b/cms/templates/widgets/sequence-edit.html index f7108e366e..c623eb4ec2 100644 --- a/cms/templates/widgets/sequence-edit.html +++ b/cms/templates/widgets/sequence-edit.html @@ -39,7 +39,7 @@ ${child.name} + data-preview-type="${child.module_class.js_module_name}">${child.url_name} handle %endfor diff --git a/lms/djangoapps/ssl_auth/__init__.py b/common/djangoapps/external_auth/__init__.py similarity index 100% rename from lms/djangoapps/ssl_auth/__init__.py rename to common/djangoapps/external_auth/__init__.py diff --git a/common/djangoapps/external_auth/admin.py b/common/djangoapps/external_auth/admin.py new file mode 100644 index 0000000000..343492bca7 --- /dev/null +++ b/common/djangoapps/external_auth/admin.py @@ -0,0 +1,8 @@ +''' +django admin pages for courseware model +''' + +from external_auth.models import * +from django.contrib import admin + +admin.site.register(ExternalAuthMap) diff --git a/common/djangoapps/external_auth/models.py b/common/djangoapps/external_auth/models.py new file mode 100644 index 0000000000..e43b306bbb --- /dev/null +++ b/common/djangoapps/external_auth/models.py @@ -0,0 +1,31 @@ +""" +WE'RE USING MIGRATIONS! + +If you make changes to this model, be sure to create an appropriate migration +file and check it in at the same time as your model changes. To do that, + +1. Go to the mitx dir +2. django-admin.py schemamigration student --auto --settings=lms.envs.dev --pythonpath=. description_of_your_change +3. Add the migration file created in mitx/common/djangoapps/external_auth/migrations/ +""" + +from django.db import models +from django.contrib.auth.models import User + +class ExternalAuthMap(models.Model): + class Meta: + unique_together = (('external_id', 'external_domain'), ) + external_id = models.CharField(max_length=255, db_index=True) + external_domain = models.CharField(max_length=255, db_index=True) + external_credentials = models.TextField(blank=True) # JSON dictionary + external_email = models.CharField(max_length=255, db_index=True) + external_name = models.CharField(blank=True,max_length=255, db_index=True) + user = models.OneToOneField(User, unique=True, db_index=True, null=True) + internal_password = models.CharField(blank=True, max_length=31) # randomly generated + dtcreated = models.DateTimeField('creation date',auto_now_add=True) + dtsignup = models.DateTimeField('signup date',null=True) # set after signup + + def __unicode__(self): + s = "[%s] = (%s / %s)" % (self.external_id, self.external_name, self.external_email) + return s + diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py new file mode 100644 index 0000000000..0425f3e158 --- /dev/null +++ b/common/djangoapps/external_auth/views.py @@ -0,0 +1,219 @@ +import json +import logging +import random +import re +import string + +from external_auth.models import ExternalAuthMap + +from django.conf import settings +from django.contrib.auth import REDIRECT_FIELD_NAME, authenticate, login +from django.contrib.auth.models import Group +from django.contrib.auth.models import User + +from django.core.urlresolvers import reverse +from django.http import HttpResponse, HttpResponseRedirect +from django.shortcuts import render_to_response +from django.shortcuts import redirect +from django.template import RequestContext +from mitxmako.shortcuts import render_to_response, render_to_string +try: + from django.views.decorators.csrf import csrf_exempt +except ImportError: + from django.contrib.csrf.middleware import csrf_exempt +from django_future.csrf import ensure_csrf_cookie +from util.cache import cache_if_anonymous + +from django_openid_auth import auth as openid_auth +from openid.consumer.consumer import (Consumer, SUCCESS, CANCEL, FAILURE) +import django_openid_auth.views as openid_views + +import student.views as student_views + +log = logging.getLogger("mitx.external_auth") + +@csrf_exempt +def default_render_failure(request, message, status=403, template_name='extauth_failure.html', exception=None): + """Render an Openid error page to the user.""" + message = "In openid_failure " + message + log.debug(message) + data = render_to_string( template_name, dict(message=message, exception=exception)) + return HttpResponse(data, status=status) + +#----------------------------------------------------------------------------- +# Openid + +def edXauth_generate_password(length=12, chars=string.letters + string.digits): + """Generate internal password for externally authenticated user""" + return ''.join([random.choice(chars) for i in range(length)]) + +@csrf_exempt +def edXauth_openid_login_complete(request, redirect_field_name=REDIRECT_FIELD_NAME, render_failure=None): + """Complete the openid login process""" + + redirect_to = request.REQUEST.get(redirect_field_name, '') + render_failure = render_failure or \ + getattr(settings, 'OPENID_RENDER_FAILURE', None) or \ + default_render_failure + + openid_response = openid_views.parse_openid_response(request) + if not openid_response: + return render_failure(request, 'This is an OpenID relying party endpoint.') + + if openid_response.status == SUCCESS: + external_id = openid_response.identity_url + oid_backend = openid_auth.OpenIDBackend() + details = oid_backend._extract_user_details(openid_response) + + log.debug('openid success, details=%s' % details) + + return edXauth_external_login_or_signup(request, + external_id, + "openid:%s" % settings.OPENID_SSO_SERVER_URL, + details, + details.get('email',''), + '%s %s' % (details.get('first_name',''),details.get('last_name','')) + ) + + return render_failure(request, 'Openid failure') + +#----------------------------------------------------------------------------- +# generic external auth login or signup + +def edXauth_external_login_or_signup(request, external_id, external_domain, credentials, email, fullname, + retfun=None): + # see if we have a map from this external_id to an edX username + try: + eamap = ExternalAuthMap.objects.get(external_id = external_id, + external_domain = external_domain, + ) + log.debug('Found eamap=%s' % eamap) + except ExternalAuthMap.DoesNotExist: + # go render form for creating edX user + eamap = ExternalAuthMap(external_id = external_id, + external_domain = external_domain, + external_credentials = json.dumps(credentials), + ) + eamap.external_email = email + eamap.external_name = fullname + eamap.internal_password = edXauth_generate_password() + log.debug('created eamap=%s' % eamap) + + eamap.save() + + internal_user = eamap.user + if internal_user is None: + log.debug('ExtAuth: no user for %s yet, doing signup' % eamap.external_email) + return edXauth_signup(request, eamap) + + uname = internal_user.username + user = authenticate(username=uname, password=eamap.internal_password) + if user is None: + log.warning("External Auth Login failed for %s / %s" % (uname,eamap.internal_password)) + return edXauth_signup(request, eamap) + + if not user.is_active: + log.warning("External Auth: user %s is not active" % (uname)) + # TODO: improve error page + return render_failure(request, 'Account not yet activated: please look for link in your email') + + login(request, user) + request.session.set_expiry(0) + student_views.try_change_enrollment(request) + log.info("Login success - {0} ({1})".format(user.username, user.email)) + if retfun is None: + return redirect('/') + return retfun() + + +#----------------------------------------------------------------------------- +# generic external auth signup + +@ensure_csrf_cookie +@cache_if_anonymous +def edXauth_signup(request, eamap=None): + """ + Present form to complete for signup via external authentication. + Even though the user has external credentials, he/she still needs + to create an account on the edX system, and fill in the user + registration form. + + eamap is an ExteralAuthMap object, specifying the external user + for which to complete the signup. + """ + + if eamap is None: + pass + + request.session['ExternalAuthMap'] = eamap # save this for use by student.views.create_account + + context = {'has_extauth_info': True, + 'show_signup_immediately' : True, + 'extauth_email': eamap.external_email, + 'extauth_username' : eamap.external_name.replace(' ',''), # default - conjoin name, no spaces + 'extauth_name': eamap.external_name, + } + + log.debug('ExtAuth: doing signup for %s' % eamap.external_email) + + return student_views.main_index(extra_context=context) + +#----------------------------------------------------------------------------- +# MIT SSL + +def ssl_dn_extract_info(dn): + ''' + Extract username, email address (may be anyuser@anydomain.com) and full name + from the SSL DN string. Return (user,email,fullname) if successful, and None + otherwise. + ''' + ss = re.search('/emailAddress=(.*)@([^/]+)', dn) + if ss: + user = ss.group(1) + email = "%s@%s" % (user, ss.group(2)) + else: + return None + ss = re.search('/CN=([^/]+)/', dn) + if ss: + fullname = ss.group(1) + else: + return None + return (user, email, fullname) + +@csrf_exempt +def edXauth_ssl_login(request): + """ + This is called by student.views.index when MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True + + Used for MIT user authentication. This presumes the web server (nginx) has been configured + to require specific client certificates. + + If the incoming protocol is HTTPS (SSL) then authenticate via client certificate. + The certificate provides user email and fullname; this populates the ExternalAuthMap. + The user is nevertheless still asked to complete the edX signup. + + Else continues on with student.views.main_index, and no authentication. + """ + certkey = "SSL_CLIENT_S_DN" # specify the request.META field to use + + cert = request.META.get(certkey,'') + if not cert: + cert = request.META.get('HTTP_'+certkey,'') + if not cert: + try: + cert = request._req.subprocess_env.get(certkey,'') # try the direct apache2 SSL key + except Exception as err: + pass + if not cert: + # no certificate information - go onward to main index + return student_views.main_index() + + (user, email, fullname) = ssl_dn_extract_info(cert) + + return edXauth_external_login_or_signup(request, + external_id=email, + external_domain="ssl:MIT", + credentials=cert, + email=email, + fullname=fullname, + retfun = student_views.main_index) diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py index eeda9a6b65..87490786c1 100644 --- a/common/djangoapps/student/views.py +++ b/common/djangoapps/student/views.py @@ -8,7 +8,6 @@ import uuid import feedparser import urllib import itertools -from collections import defaultdict from django.conf import settings from django.contrib.auth import logout, authenticate, login @@ -23,7 +22,7 @@ from django.http import HttpResponse, Http404 from django.shortcuts import redirect from mitxmako.shortcuts import render_to_response, render_to_string from django.core.urlresolvers import reverse -from BeautifulSoup import BeautifulSoup +from bs4 import BeautifulSoup from django.core.cache import cache from django_future.csrf import ensure_csrf_cookie @@ -37,6 +36,7 @@ from xmodule.modulestore.exceptions import ItemNotFoundError from models import Registration, UserProfile, PendingNameChange, PendingEmailChange, CourseEnrollment from datetime import date from collections import namedtuple +from courseware.courses import course_staff_group_name, has_staff_access_to_course, get_courses_by_university log = logging.getLogger("mitx.student") Article = namedtuple('Article', 'title url author image deck publication publish_date') @@ -60,6 +60,19 @@ def index(request): if settings.COURSEWARE_ENABLED and request.user.is_authenticated(): return redirect(reverse('dashboard')) + if settings.MITX_FEATURES.get('AUTH_USE_MIT_CERTIFICATES'): + from external_auth.views import edXauth_ssl_login + return edXauth_ssl_login(request) + + return main_index(user=request.user) + +def main_index(extra_context = {}, user=None): + ''' + Render the edX main page. + + extra_context is used to allow immediate display of certain modal windows, eg signup, + as used by external_auth. + ''' feed_data = cache.get("students_index_rss_feed_data") if feed_data == None: if hasattr(settings, 'RSS_URL'): @@ -75,13 +88,11 @@ def index(request): entry.image = soup.img['src'] if soup.img else None entry.summary = soup.getText() - universities = defaultdict(list) - courses = sorted(modulestore().get_courses(), key=lambda course: course.number) - for course in courses: - universities[course.org].append(course) - - return render_to_response('index.html', {'universities': universities, 'entries': entries}) - + # The course selection work is done in courseware.courses. + universities = get_courses_by_university(None) + context = {'universities': universities, 'entries': entries} + context.update(extra_context) + return render_to_response('index.html', context) def course_from_id(id): course_loc = CourseDescriptor.id_to_location(id) @@ -170,6 +181,14 @@ def change_enrollment(request): .format(user.username, enrollment.course_id)) return {'success': False, 'error': 'The course requested does not exist.'} + if settings.MITX_FEATURES.get('ACCESS_REQUIRE_STAFF_FOR_COURSE'): + # require that user be in the staff_* group (or be an overall admin) to be able to enroll + # eg staff_6.002x or staff_6.00x + if not has_staff_access_to_course(user,course): + staff_group = course_staff_group_name(course) + log.debug('user %s denied enrollment to %s ; not in %s' % (user,course.location.url(),staff_group)) + return {'success': False, 'error' : '%s membership required to access course.' % staff_group} + enrollment, created = CourseEnrollment.objects.get_or_create(user=user, course_id=course.id) return {'success': True} @@ -256,11 +275,26 @@ def change_setting(request): @ensure_csrf_cookie def create_account(request, post_override=None): - ''' JSON call to enroll in the course. ''' + ''' + JSON call to create new edX account. + Used by form in signup_modal.html, which is included into navigation.html + ''' js = {'success': False} post_vars = post_override if post_override else request.POST + # if doing signup for an external authorization, then get email, password, name from the eamap + # don't use the ones from the form, since the user could have hacked those + DoExternalAuth = 'ExternalAuthMap' in request.session + if DoExternalAuth: + eamap = request.session['ExternalAuthMap'] + email = eamap.external_email + name = eamap.external_name + password = eamap.internal_password + post_vars = dict(post_vars.items()) + post_vars.update(dict(email=email, name=name, password=password)) + log.debug('extauth test: post_vars = %s' % post_vars) + # Confirm we have a properly formed request for a in ['username', 'email', 'password', 'name']: if a not in post_vars: @@ -355,8 +389,9 @@ def create_account(request, post_override=None): 'key': r.activation_key, } + # composes activation email subject = render_to_string('emails/activation_email_subject.txt', d) - # Email subject *must not* contain newlines + # Email subject *must not* contain newlines subject = ''.join(subject.splitlines()) message = render_to_string('emails/activation_email.txt', d) @@ -381,6 +416,17 @@ def create_account(request, post_override=None): try_change_enrollment(request) + if DoExternalAuth: + eamap.user = login_user + eamap.dtsignup = datetime.datetime.now() + eamap.save() + log.debug('Updated ExternalAuthMap for %s to be %s' % (post_vars['username'],eamap)) + + if settings.MITX_FEATURES.get('BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'): + log.debug('bypassing activation email') + login_user.is_active = True + login_user.save() + js = {'success': True} return HttpResponse(json.dumps(js), mimetype="application/json") diff --git a/common/djangoapps/track/models.py b/common/djangoapps/track/models.py index 71a8362390..401fa2832f 100644 --- a/common/djangoapps/track/models.py +++ b/common/djangoapps/track/models.py @@ -1,3 +1,21 @@ from django.db import models -# Create your models here. +from django.db import models + +class TrackingLog(models.Model): + dtcreated = models.DateTimeField('creation date',auto_now_add=True) + username = models.CharField(max_length=32,blank=True) + ip = models.CharField(max_length=32,blank=True) + event_source = models.CharField(max_length=32) + event_type = models.CharField(max_length=32,blank=True) + event = models.TextField(blank=True) + agent = models.CharField(max_length=256,blank=True) + page = models.CharField(max_length=32,blank=True,null=True) + time = models.DateTimeField('event time') + + def __unicode__(self): + s = "[%s] %s@%s: %s | %s | %s | %s" % (self.time, self.username, self.ip, self.event_source, + self.event_type, self.page, self.event) + return s + + diff --git a/common/djangoapps/track/views.py b/common/djangoapps/track/views.py index a60d8bef28..b5f9c54665 100644 --- a/common/djangoapps/track/views.py +++ b/common/djangoapps/track/views.py @@ -2,19 +2,32 @@ import json import logging import os import datetime +import dateutil.parser -# Create your views here. +from django.contrib.auth.decorators import login_required from django.http import HttpResponse from django.http import Http404 +from django.shortcuts import redirect from django.conf import settings +from mitxmako.shortcuts import render_to_response + +from django_future.csrf import ensure_csrf_cookie +from track.models import TrackingLog log = logging.getLogger("tracking") +LOGFIELDS = ['username','ip','event_source','event_type','event','agent','page','time'] def log_event(event): event_str = json.dumps(event) log.info(event_str[:settings.TRACK_MAX_EVENT]) - + if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'): + event['time'] = dateutil.parser.parse(event['time']) + tldat = TrackingLog(**dict( (x,event[x]) for x in LOGFIELDS )) + try: + tldat.save() + except Exception as err: + log.exception(err) def user_track(request): try: # TODO: Do the same for many of the optional META parameters @@ -70,4 +83,16 @@ def server_track(request, event_type, event, page=None): "page": page, "time": datetime.datetime.utcnow().isoformat(), } + + if event_type=="/event_logs" and request.user.is_staff: # don't log + return log_event(event) + +@login_required +@ensure_csrf_cookie +def view_tracking_log(request): + if not request.user.is_staff: + return redirect('/') + record_instances = TrackingLog.objects.all().order_by('-time')[0:100] + return render_to_response('tracking_log.html',{'records':record_instances}) + diff --git a/common/djangoapps/xmodule_modifiers.py b/common/djangoapps/xmodule_modifiers.py index d0d1e0be15..843d2eaa38 100644 --- a/common/djangoapps/xmodule_modifiers.py +++ b/common/djangoapps/xmodule_modifiers.py @@ -1,9 +1,15 @@ +import re import json +import logging + from django.conf import settings from functools import wraps from static_replace import replace_urls from mitxmako.shortcuts import render_to_string +from xmodule.seq_module import SequenceModule +from xmodule.vertical_module import VerticalModule +log = logging.getLogger("mitx.xmodule_modifiers") def wrap_xmodule(get_html, module, template): """ @@ -69,29 +75,32 @@ def add_histogram(get_html, module): the output of the old get_html function with additional information for admin users only, including a histogram of student answers and the definition of the xmodule + + Does nothing if module is a SequenceModule """ @wraps(get_html) def _get_html(): + + if type(module) in [SequenceModule, VerticalModule]: # TODO: make this more general, eg use an XModule attribute instead + return get_html() + module_id = module.id histogram = grade_histogram(module_id) render_histogram = len(histogram) > 0 - # TODO: fixme - no filename in module.xml in general (this code block for edx4edx) - # the following if block is for summer 2012 edX course development; it will change when the CMS comes online - if settings.MITX_FEATURES.get('DISPLAY_EDIT_LINK') and settings.DEBUG and module_xml.get('filename') is not None: - coursename = multicourse_settings.get_coursename_from_request(request) - github_url = multicourse_settings.get_course_github_url(coursename) - fn = module_xml.get('filename') - if module_xml.tag=='problem': fn = 'problems/' + fn # grrr - edit_link = (github_url + '/tree/master/' + fn) if github_url is not None else None - if module_xml.tag=='problem': edit_link += '.xml' # grrr + # TODO (ichuang): Remove after fall 2012 LMS migration done + if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'): + [filepath, filename] = module.definition.get('filename','') + osfs = module.system.filestore + if osfs.exists(filename): + filepath = filename # if original, unmangled filename exists then use it (github doesn't like symlinks) + data_dir = osfs.root_path.rsplit('/')[-1] + edit_link = "https://github.com/MITx/%s/tree/master/%s" % (data_dir,filepath) else: edit_link = False - # Cast module.definition and module.metadata to dicts so that json can dump them - # even though they are lazily loaded - staff_context = {'definition': json.dumps(dict(module.definition), indent=4), - 'metadata': json.dumps(dict(module.metadata), indent=4), + staff_context = {'definition': module.definition.get('data'), + 'metadata': json.dumps(module.metadata, indent=4), 'element_id': module.location.html_id(), 'edit_link': edit_link, 'histogram': json.dumps(histogram), @@ -100,3 +109,4 @@ def add_histogram(get_html, module): return render_to_string("staff_problem_info.html", staff_context) return _get_html + diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index d5f45831ee..ba99ee681e 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -294,20 +294,30 @@ class LoncapaProblem(object): try: ifp = self.system.filestore.open(file) # open using ModuleSystem OSFS filestore except Exception as err: - log.error('Error %s in problem xml include: %s' % (err, etree.tostring(inc, pretty_print=True))) - log.error('Cannot find file %s in %s' % (file, self.system.filestore)) - if not self.system.get('DEBUG'): # if debugging, don't fail - just log error + log.error('Error %s in problem xml include: %s' % ( + err, etree.tostring(inc, pretty_print=True))) + log.error('Cannot find file %s in %s' % ( + file, self.system.filestore)) + # if debugging, don't fail - just log error + # TODO (vshnayder): need real error handling, display to users + if not self.system.get('DEBUG'): raise - else: continue + else: + continue try: incxml = etree.XML(ifp.read()) # read in and convert to XML except Exception as err: - log.error('Error %s in problem xml include: %s' % (err, etree.tostring(inc, pretty_print=True))) + log.error('Error %s in problem xml include: %s' % ( + err, etree.tostring(inc, pretty_print=True))) log.error('Cannot parse XML in %s' % (file)) - if not self.system.get('DEBUG'): # if debugging, don't fail - just log error + # if debugging, don't fail - just log error + # TODO (vshnayder): same as above + if not self.system.get('DEBUG'): raise - else: continue - parent = inc.getparent() # insert new XML into tree in place of inlcude + else: + continue + # insert new XML into tree in place of inlcude + parent = inc.getparent() parent.insert(parent.index(inc), incxml) parent.remove(inc) log.debug('Included %s into %s' % (file, self.problem_id)) @@ -335,7 +345,7 @@ class LoncapaProblem(object): # path is an absolute path or a path relative to the data dir dir = os.path.join(self.system.filestore.root_path, dir) abs_dir = os.path.normpath(dir) - log.debug("appending to path: %s" % abs_dir) + #log.debug("appending to path: %s" % abs_dir) path.append(abs_dir) return path diff --git a/common/lib/supertrace.py b/common/lib/supertrace.py new file mode 100644 index 0000000000..e17cd7a8ba --- /dev/null +++ b/common/lib/supertrace.py @@ -0,0 +1,52 @@ +""" +A handy util to print a django-debug-screen-like stack trace with +values of local variables. +""" + +import sys, traceback +from django.utils.encoding import smart_unicode + + +def supertrace(max_len=160): + """ + Print the usual traceback information, followed by a listing of all the + local variables in each frame. Should be called from an exception handler. + + if max_len is not None, will print up to max_len chars for each local variable. + + (cite: modified from somewhere on stackoverflow) + """ + tb = sys.exc_info()[2] + while True: + if not tb.tb_next: + break + tb = tb.tb_next + stack = [] + frame = tb.tb_frame + while frame: + stack.append(f) + frame = frame.f_back + stack.reverse() + # First print the regular traceback + traceback.print_exc() + + print "Locals by frame, innermost last" + for frame in stack: + print + print "Frame %s in %s at line %s" % (frame.f_code.co_name, + frame.f_code.co_filename, + frame.f_lineno) + for key, value in frame.f_locals.items(): + print ("\t%20s = " % smart_unicode(key, errors='ignore')), + # We have to be careful not to cause a new error in our error + # printer! Calling str() on an unknown object could cause an + # error. + try: + s = smart_unicode(value, errors='ignore') + if max_len is not None: + s = s[:max_len] + print s + except: + print "" + + diff --git a/common/lib/xmodule/progress.py b/common/lib/xmodule/progress.py deleted file mode 100644 index 70c8ec9da1..0000000000 --- a/common/lib/xmodule/progress.py +++ /dev/null @@ -1,157 +0,0 @@ -''' -Progress class for modules. Represents where a student is in a module. - -Useful things to know: - - Use Progress.to_js_status_str() to convert a progress into a simple - status string to pass to js. - - Use Progress.to_js_detail_str() to convert a progress into a more detailed - string to pass to js. - -In particular, these functions have a canonical handing of None. - -For most subclassing needs, you should only need to reimplement -frac() and __str__(). -''' - -from collections import namedtuple -import numbers - - -class Progress(object): - '''Represents a progress of a/b (a out of b done) - - a and b must be numeric, but not necessarily integer, with - 0 <= a <= b and b > 0. - - Progress can only represent Progress for modules where that makes sense. Other - modules (e.g. html) should return None from get_progress(). - - TODO: add tag for module type? Would allow for smarter merging. - ''' - - def __init__(self, a, b): - '''Construct a Progress object. a and b must be numbers, and must have - 0 <= a <= b and b > 0 - ''' - - # Want to do all checking at construction time, so explicitly check types - if not (isinstance(a, numbers.Number) and - isinstance(b, numbers.Number)): - raise TypeError('a and b must be numbers. Passed {0}/{1}'.format(a, b)) - - if not (0 <= a <= b and b > 0): - raise ValueError( - 'fraction a/b = {0}/{1} must have 0 <= a <= b and b > 0'.format(a, b)) - - self._a = a - self._b = b - - def frac(self): - ''' Return tuple (a,b) representing progress of a/b''' - return (self._a, self._b) - - def percent(self): - ''' Returns a percentage progress as a float between 0 and 100. - - subclassing note: implemented in terms of frac(), assumes sanity - checking is done at construction time. - ''' - (a, b) = self.frac() - return 100.0 * a / b - - def started(self): - ''' Returns True if fractional progress is greater than 0. - - subclassing note: implemented in terms of frac(), assumes sanity - checking is done at construction time. - ''' - return self.frac()[0] > 0 - - def inprogress(self): - ''' Returns True if fractional progress is strictly between 0 and 1. - - subclassing note: implemented in terms of frac(), assumes sanity - checking is done at construction time. - ''' - (a, b) = self.frac() - return a > 0 and a < b - - def done(self): - ''' Return True if this represents done. - - subclassing note: implemented in terms of frac(), assumes sanity - checking is done at construction time. - ''' - (a, b) = self.frac() - return a == b - - def ternary_str(self): - ''' Return a string version of this progress: either - "none", "in_progress", or "done". - - subclassing note: implemented in terms of frac() - ''' - (a, b) = self.frac() - if a == 0: - return "none" - if a < b: - return "in_progress" - return "done" - - def __eq__(self, other): - ''' Two Progress objects are equal if they have identical values. - Implemented in terms of frac()''' - if not isinstance(other, Progress): - return False - (a, b) = self.frac() - (a2, b2) = other.frac() - return a == a2 and b == b2 - - def __ne__(self, other): - ''' The opposite of equal''' - return not self.__eq__(other) - - def __str__(self): - ''' Return a string representation of this string. - - subclassing note: implemented in terms of frac(). - ''' - (a, b) = self.frac() - return "{0}/{1}".format(a, b) - - @staticmethod - def add_counts(a, b): - '''Add two progress indicators, assuming that each represents items done: - (a / b) + (c / d) = (a + c) / (b + d). - If either is None, returns the other. - ''' - if a is None: - return b - if b is None: - return a - # get numerators + denominators - (n, d) = a.frac() - (n2, d2) = b.frac() - return Progress(n + n2, d + d2) - - @staticmethod - def to_js_status_str(progress): - ''' - Return the "status string" version of the passed Progress - object that should be passed to js. Use this function when - sending Progress objects to js to limit dependencies. - ''' - if progress is None: - return "NA" - return progress.ternary_str() - - @staticmethod - def to_js_detail_str(progress): - ''' - Return the "detail string" version of the passed Progress - object that should be passed to js. Use this function when - passing Progress objects to js to limit dependencies. - ''' - if progress is None: - return "NA" - return str(progress) diff --git a/common/lib/xmodule/setup.py b/common/lib/xmodule/setup.py index b495cd0aee..8a0a6bb139 100644 --- a/common/lib/xmodule/setup.py +++ b/common/lib/xmodule/setup.py @@ -25,6 +25,7 @@ setup( "discuss = xmodule.backcompat_module:TranslateCustomTagDescriptor", "html = xmodule.html_module:HtmlDescriptor", "image = xmodule.backcompat_module:TranslateCustomTagDescriptor", + "error = xmodule.error_module:ErrorDescriptor", "problem = xmodule.capa_module:CapaDescriptor", "problemset = xmodule.vertical_module:VerticalDescriptor", "section = xmodule.backcompat_module:SemanticSectionDescriptor", diff --git a/common/lib/xmodule/xmodule/backcompat_module.py b/common/lib/xmodule/xmodule/backcompat_module.py index 997ad476c4..c49f23b99e 100644 --- a/common/lib/xmodule/xmodule/backcompat_module.py +++ b/common/lib/xmodule/xmodule/backcompat_module.py @@ -32,21 +32,25 @@ def process_includes(fn): # read in and convert to XML incxml = etree.XML(ifp.read()) - # insert new XML into tree in place of inlcude + # insert new XML into tree in place of include parent.insert(parent.index(next_include), incxml) except Exception: - msg = "Error in problem xml include: %s" % (etree.tostring(next_include, pretty_print=True)) - log.exception(msg) - parent = next_include.getparent() + # Log error + msg = "Error in problem xml include: %s" % ( + etree.tostring(next_include, pretty_print=True)) + # tell the tracker + system.error_tracker(msg) + # work around + parent = next_include.getparent() errorxml = etree.Element('error') messagexml = etree.SubElement(errorxml, 'message') messagexml.text = msg stackxml = etree.SubElement(errorxml, 'stacktrace') stackxml.text = traceback.format_exc() - # insert error XML in place of include parent.insert(parent.index(next_include), errorxml) + parent.remove(next_include) next_include = xml_object.find('include') diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py index 2bf1dd0487..7ce76def32 100644 --- a/common/lib/xmodule/xmodule/capa_module.py +++ b/common/lib/xmodule/xmodule/capa_module.py @@ -5,6 +5,7 @@ import json import logging import traceback import re +import sys from datetime import timedelta from lxml import etree @@ -92,7 +93,8 @@ class CapaModule(XModule): display_due_date_string = self.metadata.get('due', None) if display_due_date_string is not None: self.display_due_date = dateutil.parser.parse(display_due_date_string) - #log.debug("Parsed " + display_due_date_string + " to " + str(self.display_due_date)) + #log.debug("Parsed " + display_due_date_string + + # " to " + str(self.display_due_date)) else: self.display_due_date = None @@ -100,7 +102,8 @@ class CapaModule(XModule): if grace_period_string is not None and self.display_due_date: self.grace_period = parse_timedelta(grace_period_string) self.close_date = self.display_due_date + self.grace_period - #log.debug("Then parsed " + grace_period_string + " to closing date" + str(self.close_date)) + #log.debug("Then parsed " + grace_period_string + + # " to closing date" + str(self.close_date)) else: self.grace_period = None self.close_date = self.display_due_date @@ -139,10 +142,16 @@ class CapaModule(XModule): try: self.lcp = LoncapaProblem(self.definition['data'], self.location.html_id(), instance_state, seed=seed, system=self.system) - except Exception: - msg = 'cannot create LoncapaProblem %s' % self.location.url() - log.exception(msg) + except Exception as err: + msg = 'cannot create LoncapaProblem {loc}: {err}'.format( + loc=self.location.url(), err=err) + # TODO (vshnayder): do modules need error handlers too? + # We shouldn't be switching on DEBUG. if self.system.DEBUG: + log.error(msg) + # TODO (vshnayder): This logic should be general, not here--and may + # want to preserve the data instead of replacing it. + # e.g. in the CMS msg = '

      %s

      ' % msg.replace('<', '<') msg += '

      %s

      ' % traceback.format_exc().replace('<', '<') # create a dummy problem with error message instead of failing @@ -153,7 +162,8 @@ class CapaModule(XModule): problem_text, self.location.html_id(), instance_state, seed=seed, system=self.system) else: - raise + # add extra info and raise + raise Exception(msg), None, sys.exc_info()[2] @property def rerandomize(self): @@ -192,6 +202,7 @@ class CapaModule(XModule): try: return Progress(score, total) except Exception as err: + # TODO (vshnayder): why is this still here? still needed? if self.system.DEBUG: return None raise @@ -211,6 +222,7 @@ class CapaModule(XModule): try: html = self.lcp.get_html() except Exception, err: + # TODO (vshnayder): another switch on DEBUG. if self.system.DEBUG: log.exception(err) msg = ( @@ -560,6 +572,7 @@ class CapaDescriptor(RawDescriptor): 'problems/' + path[8:], path[8:], ] + @classmethod def split_to_file(cls, xml_object): '''Problems always written in their own files''' diff --git a/common/lib/xmodule/xmodule/course_module.py b/common/lib/xmodule/xmodule/course_module.py index dfac1ac9c6..acdc574220 100644 --- a/common/lib/xmodule/xmodule/course_module.py +++ b/common/lib/xmodule/xmodule/course_module.py @@ -1,7 +1,9 @@ +from fs.errors import ResourceNotFoundError import time import dateutil.parser import logging +from xmodule.graders import load_grading_policy from xmodule.modulestore import Location from xmodule.seq_module import SequenceDescriptor, SequenceModule @@ -14,20 +16,57 @@ class CourseDescriptor(SequenceDescriptor): def __init__(self, system, definition=None, **kwargs): super(CourseDescriptor, self).__init__(system, definition, **kwargs) + + self._grader = None + self._grade_cutoffs = None + msg = None try: self.start = time.strptime(self.metadata["start"], "%Y-%m-%dT%H:%M") except KeyError: self.start = time.gmtime(0) #The epoch - log.critical("Course loaded without a start date. %s", self.id) + msg = "Course loaded without a start date. id = %s" % self.id + log.critical(msg) except ValueError as e: self.start = time.gmtime(0) #The epoch - log.critical("Course loaded with a bad start date. %s '%s'", - self.id, e) + msg = "Course loaded with a bad start date. %s '%s'" % (self.id, e) + log.critical(msg) + + # Don't call the tracker from the exception handler. + if msg is not None: + system.error_tracker(msg) + def has_started(self): return time.gmtime() > self.start - + + @property + def grader(self): + self.__load_grading_policy() + return self._grader + + @property + def grade_cutoffs(self): + self.__load_grading_policy() + return self._grade_cutoffs + + + def __load_grading_policy(self): + if not self._grader or not self._grade_cutoffs: + policy_string = "" + + try: + with self.system.resources_fs.open("grading_policy.json") as grading_policy_file: + policy_string = grading_policy_file.read() + except (IOError, ResourceNotFoundError): + log.warning("Unable to load course settings file from grading_policy.json in course " + self.id) + + grading_policy = load_grading_policy(policy_string) + + self._grader = grading_policy['GRADER'] + self._grade_cutoffs = grading_policy['GRADE_CUTOFFS'] + + @staticmethod def id_to_location(course_id): '''Convert the given course_id (org/course/name) to a location object. @@ -72,3 +111,4 @@ class CourseDescriptor(SequenceDescriptor): @property def org(self): return self.location.org + diff --git a/common/lib/xmodule/xmodule/css/sequence/display.scss b/common/lib/xmodule/xmodule/css/sequence/display.scss index d2ea986a4c..6e7c5d24a7 100644 --- a/common/lib/xmodule/xmodule/css/sequence/display.scss +++ b/common/lib/xmodule/xmodule/css/sequence/display.scss @@ -2,9 +2,8 @@ nav.sequence-nav { // TODO (cpennington): This doesn't work anymore. XModules aren't able to // import from external sources. @extend .topbar; - - border-bottom: 1px solid darken($cream, 20%); - margin-bottom: $body-line-height; + border-bottom: 1px solid $border-color; + margin: (-(lh())) (-(lh())) lh() (-(lh())); position: relative; @include border-top-right-radius(4px); @@ -12,6 +11,8 @@ nav.sequence-nav { @include box-sizing(border-box); display: table; height: 100%; + margin: 0; + padding-left: 3px; padding-right: flex-grid(1, 9); width: 100%; @@ -20,135 +21,104 @@ nav.sequence-nav { } li { - border-left: 1px solid darken($cream, 20%); display: table-cell; min-width: 20px; - &:first-child { - border-left: none; - } - - .inactive { - background-repeat: no-repeat; - - &:hover { - background-color: lighten($cream, 3%); - } - } - - .visited { - background-color: #DCCDA2; - background-repeat: no-repeat; - @include box-shadow(inset 0 0 3px darken(#dccda2, 10%)); - - &:hover { - background-color: $cream; - background-position: center center; - } - } - - .active { - background-color: #fff; - background-repeat: no-repeat; - @include box-shadow(0 1px 0 #fff); - - &:hover { - background-color: #fff; - background-position: center; - } - } - a { - background-position: center center; - border: none; + background-position: center; + background-repeat: no-repeat; + border: 1px solid transparent; + border-bottom: none; + @include border-radius(3px 3px 0 0); cursor: pointer; display: block; - height: 17px; + height: 10px; padding: 15px 0 14px; position: relative; - @include transition(all, .4s, $ease-in-out-quad); + @include transition(); width: 100%; - &.progress { - border-bottom-style: solid; - border-bottom-width: 4px; + &:hover { + background-repeat: no-repeat; + background-position: center; + background-color: #F6F6F6; + } + + &.visited { + background-color: #F6F6F6; + + &:hover { + background-position: center center; + } + } + + &.active { + border-color: $border-color; + @include box-shadow(0 2px 0 #fff); + background-color: #fff; + z-index: 9; + + &:hover { + background-position: center; + background-color: #fff; + } } &.progress-none { - @extend .progress; - border-bottom-color: red; + background-color: lighten(red, 50%); } &.progress-some { - @extend .progress; - border-bottom-color: yellow; + background-color: yellow; } &.progress-done { - @extend .progress; - border-bottom-color: green; + background-color: green; } //video &.seq_video { &.inactive { - @extend .inactive; background-image: url('../images/sequence-nav/video-icon-normal.png'); - background-position: center; } &.visited { - @extend .visited; background-image: url('../images/sequence-nav/video-icon-visited.png'); - background-position: center; } &.active { @extend .active; background-image: url('../images/sequence-nav/video-icon-current.png'); - background-position: center; } } //other &.seq_other { &.inactive { - @extend .inactive; background-image: url('../images/sequence-nav/document-icon-normal.png'); - background-position: center; } &.visited { - @extend .visited; background-image: url('../images/sequence-nav/document-icon-visited.png'); - background-position: center; } &.active { - @extend .active; background-image: url('../images/sequence-nav/document-icon-current.png'); - background-position: center; } } //vertical & problems &.seq_vertical, &.seq_problem { &.inactive { - @extend .inactive; background-image: url('../images/sequence-nav/list-icon-normal.png'); - background-position: center; } &.visited { - @extend .visited; background-image: url('../images/sequence-nav/list-icon-visited.png'); - background-position: center; } &.active { - @extend .active; background-image: url('../images/sequence-nav/list-icon-current.png'); - background-position: center; } } @@ -156,6 +126,7 @@ nav.sequence-nav { background: #333; color: #fff; display: none; + font-family: $sans-serif; line-height: lh(); left: 0px; opacity: 0; @@ -206,27 +177,29 @@ nav.sequence-nav { right: 0; top: 0; width: flex-grid(1, 9); + border: 1px solid $border-color; + border-bottom: 0; + @include border-radius(3px 3px 0 0); li { float: left; + margin-bottom: 0; width: 50%; &.prev, &.next { a { - background-color: darken($cream, 5%); - background-position: center center; + background-position: center; background-repeat: no-repeat; - border-left: 1px solid darken(#f6efd4, 20%); - @include box-shadow(inset 1px 0 0 lighten(#f6efd4, 5%)); - @include box-sizing(border-box); - cursor: pointer; display: block; + height: 10px; + padding: 15px 0 14px; text-indent: -9999px; @include transition(all, .2s, $ease-in-out-quad); &:hover { opacity: .5; + background-color: #f4f4f4; } &.disabled { @@ -239,20 +212,13 @@ nav.sequence-nav { &.prev { a { background-image: url('../images/sequence-nav/previous-icon.png'); - - &:hover { - background-color: $cream; - } } } &.next { a { + border-left: 1px solid lighten($border-color, 10%); background-image: url('../images/sequence-nav/next-icon.png'); - - &:hover { - background-color: $cream; - } } } } @@ -273,11 +239,8 @@ nav.sequence-bottom { ul { @extend .clearfix; - background-color: darken(#F6EFD4, 5%); - background-color: darken($cream, 5%); - border: 1px solid darken(#f6efd4, 20%); + border: 1px solid $border-color; @include border-radius(3px); - @include box-shadow(inset 0 0 0 1px lighten(#f6efd4, 5%)); @include inline-block(); li { @@ -297,14 +260,13 @@ nav.sequence-bottom { width: 45px; &:hover { - background-color: $cream; - color: darken($cream, 60%); + background-color: #ddd; + color: #000; opacity: .5; text-decoration: none; } &.disabled { - background-color: lighten($cream, 10%); opacity: .4; } } @@ -313,7 +275,7 @@ nav.sequence-bottom { &.prev { a { background-image: url('../images/sequence-nav/previous-icon.png'); - border-right: 1px solid darken(#f6efd4, 20%); + border-right: 1px solid lighten($border-color, 10%); &:hover { background-color: none; diff --git a/common/lib/xmodule/xmodule/css/video/display.scss b/common/lib/xmodule/xmodule/css/video/display.scss index b7187cff26..789a267755 100644 --- a/common/lib/xmodule/xmodule/css/video/display.scss +++ b/common/lib/xmodule/xmodule/css/video/display.scss @@ -114,14 +114,13 @@ div.video { @extend .dullify; float: left; list-style: none; - margin-right: lh(); + margin: 0 lh() 0 0; padding: 0; li { float: left; margin-bottom: 0; - a { border-bottom: none; border-right: 1px solid #000; @@ -183,6 +182,8 @@ div.video { ol.video_speeds { display: block; opacity: 1; + padding: 0; + margin: 0; } } @@ -210,6 +211,7 @@ div.video { font-weight: normal; letter-spacing: 1px; padding: 0 lh(.25) 0 lh(.5); + line-height: 46px; text-transform: uppercase; } @@ -218,6 +220,7 @@ div.video { font-weight: bold; margin-bottom: 0; padding: 0 lh(.5) 0 0; + line-height: 46px; } &:hover, &:active, &:focus { @@ -422,10 +425,12 @@ div.video { } ol.subtitles { + padding-left: 0; float: left; max-height: 460px; overflow: auto; width: flex-grid(3, 9); + margin: 0; li { border: 0; diff --git a/common/lib/xmodule/xmodule/editing_module.py b/common/lib/xmodule/xmodule/editing_module.py new file mode 100644 index 0000000000..67a4d66dad --- /dev/null +++ b/common/lib/xmodule/xmodule/editing_module.py @@ -0,0 +1,29 @@ +from pkg_resources import resource_string +from lxml import etree +from xmodule.mako_module import MakoModuleDescriptor +import logging + +log = logging.getLogger(__name__) + +class EditingDescriptor(MakoModuleDescriptor): + """ + Module that provides a raw editing view of its data and children. It does not + perform any validation on its definition---just passes it along to the browser. + + This class is intended to be used as a mixin. + """ + mako_template = "widgets/raw-edit.html" + + js = {'coffee': [resource_string(__name__, 'js/src/raw/edit.coffee')]} + js_module_name = "RawDescriptor" + + def get_context(self): + return { + 'module': self, + 'data': self.definition.get('data', ''), + # TODO (vshnayder): allow children and metadata to be edited. + #'children' : self.definition.get('children, ''), + + # TODO: show both own metadata and inherited? + #'metadata' : self.own_metadata, + } diff --git a/common/lib/xmodule/xmodule/error_module.py b/common/lib/xmodule/xmodule/error_module.py new file mode 100644 index 0000000000..ecc90873b9 --- /dev/null +++ b/common/lib/xmodule/xmodule/error_module.py @@ -0,0 +1,89 @@ +import sys +import logging + +from pkg_resources import resource_string +from lxml import etree +from xmodule.x_module import XModule +from xmodule.mako_module import MakoModuleDescriptor +from xmodule.xml_module import XmlDescriptor +from xmodule.editing_module import EditingDescriptor +from xmodule.errortracker import exc_info_to_str + + +log = logging.getLogger(__name__) + +class ErrorModule(XModule): + def get_html(self): + '''Show an error. + TODO (vshnayder): proper style, divs, etc. + ''' + # staff get to see all the details + return self.system.render_template('module-error.html', { + 'data' : self.definition['data']['contents'], + 'error' : self.definition['data']['error_msg'], + 'is_staff' : self.system.is_staff, + }) + +class ErrorDescriptor(EditingDescriptor): + """ + Module that provides a raw editing view of broken xml. + """ + module_class = ErrorModule + + @classmethod + def from_xml(cls, xml_data, system, org=None, course=None, + error_msg='Error not available'): + '''Create an instance of this descriptor from the supplied data. + + Does not try to parse the data--just stores it. + + Takes an extra, optional, parameter--the error that caused an + issue. (should be a string, or convert usefully into one). + ''' + # Use a nested inner dictionary because 'data' is hardcoded + inner = {} + definition = {'data': inner} + inner['error_msg'] = str(error_msg) + + try: + # If this is already an error tag, don't want to re-wrap it. + xml_obj = etree.fromstring(xml_data) + if xml_obj.tag == 'error': + xml_data = xml_obj.text + error_node = xml_obj.find('error_msg') + if error_node is not None: + inner['error_msg'] = error_node.text + else: + inner['error_msg'] = 'Error not available' + + except etree.XMLSyntaxError: + # Save the error to display later--overrides other problems + inner['error_msg'] = exc_info_to_str(sys.exc_info()) + + inner['contents'] = xml_data + # TODO (vshnayder): Do we need a unique slug here? Just pick a random + # 64-bit num? + location = ['i4x', org, course, 'error', 'slug'] + metadata = {} # stays in the xml_data + + return cls(system, definition, location=location, metadata=metadata) + + def export_to_xml(self, resource_fs): + ''' + If the definition data is invalid xml, export it wrapped in an "error" + tag. If it is valid, export without the wrapper. + + NOTE: There may still be problems with the valid xml--it could be + missing required attributes, could have the wrong tags, refer to missing + files, etc. That would just get re-wrapped on import. + ''' + try: + xml = etree.fromstring(self.definition['data']['contents']) + return etree.tostring(xml) + except etree.XMLSyntaxError: + # still not valid. + root = etree.Element('error') + root.text = self.definition['data']['contents'] + err_node = etree.SubElement(root, 'error_msg') + err_node.text = self.definition['data']['error_msg'] + return etree.tostring(root) diff --git a/common/lib/xmodule/xmodule/errorhandlers.py b/common/lib/xmodule/xmodule/errorhandlers.py deleted file mode 100644 index 0f97377b2a..0000000000 --- a/common/lib/xmodule/xmodule/errorhandlers.py +++ /dev/null @@ -1,45 +0,0 @@ -import logging -import sys - -log = logging.getLogger(__name__) - -def in_exception_handler(): - '''Is there an active exception?''' - return sys.exc_info() != (None, None, None) - -def strict_error_handler(msg, exc_info=None): - ''' - Do not let errors pass. If exc_info is not None, ignore msg, and just - re-raise. Otherwise, check if we are in an exception-handling context. - If so, re-raise. Otherwise, raise Exception(msg). - - Meant for use in validation, where any errors should trap. - ''' - if exc_info is not None: - raise exc_info[0], exc_info[1], exc_info[2] - - if in_exception_handler(): - raise - - raise Exception(msg) - - -def logging_error_handler(msg, exc_info=None): - '''Log all errors, but otherwise let them pass, relying on the caller to - workaround.''' - if exc_info is not None: - log.exception(msg, exc_info=exc_info) - return - - if in_exception_handler(): - log.exception(msg) - return - - log.error(msg) - - -def ignore_errors_handler(msg, exc_info=None): - '''Ignore all errors, relying on the caller to workaround. - Meant for use in the LMS, where an error in one part of the course - shouldn't bring down the whole system''' - pass diff --git a/common/lib/xmodule/xmodule/errortracker.py b/common/lib/xmodule/xmodule/errortracker.py new file mode 100644 index 0000000000..8ac2903149 --- /dev/null +++ b/common/lib/xmodule/xmodule/errortracker.py @@ -0,0 +1,44 @@ +import logging +import sys +import traceback + +from collections import namedtuple + +log = logging.getLogger(__name__) + +ErrorLog = namedtuple('ErrorLog', 'tracker errors') + +def exc_info_to_str(exc_info): + """Given some exception info, convert it into a string using + the traceback.format_exception() function. + """ + return ''.join(traceback.format_exception(*exc_info)) + +def in_exception_handler(): + '''Is there an active exception?''' + return sys.exc_info() != (None, None, None) + + +def make_error_tracker(): + '''Return an ErrorLog (named tuple), with fields (tracker, errors), where + the logger appends a tuple (message, exception_str) to the errors on every + call. exception_str is in the format returned by traceback.format_exception. + + error_list is a simple list. If the caller modifies it, info + will be lost. + ''' + errors = [] + + def error_tracker(msg): + '''Log errors''' + exc_str = '' + if in_exception_handler(): + exc_str = exc_info_to_str(sys.exc_info()) + + errors.append((msg, exc_str)) + + return ErrorLog(error_tracker, errors) + +def null_error_tracker(msg): + '''A dummy error tracker that just ignores the messages''' + pass diff --git a/common/lib/xmodule/xmodule/graders.py b/common/lib/xmodule/xmodule/graders.py index 4473c8f1b9..fca862aa9f 100644 --- a/common/lib/xmodule/xmodule/graders.py +++ b/common/lib/xmodule/xmodule/graders.py @@ -1,4 +1,5 @@ import abc +import json import logging from collections import namedtuple @@ -9,6 +10,69 @@ log = logging.getLogger("mitx.courseware") # Section either indicates the name of the problem or the name of the section Score = namedtuple("Score", "earned possible graded section") +def load_grading_policy(course_policy_string): + """ + This loads a grading policy from a string (usually read from a file), + which can be a JSON object or an empty string. + + The JSON object can have the keys GRADER and GRADE_CUTOFFS. If either is + missing, it reverts to the default. + """ + + default_policy_string = """ + { + "GRADER" : [ + { + "type" : "Homework", + "min_count" : 12, + "drop_count" : 2, + "short_label" : "HW", + "weight" : 0.15 + }, + { + "type" : "Lab", + "min_count" : 12, + "drop_count" : 2, + "category" : "Labs", + "weight" : 0.15 + }, + { + "type" : "Midterm", + "name" : "Midterm Exam", + "short_label" : "Midterm", + "weight" : 0.3 + }, + { + "type" : "Final", + "name" : "Final Exam", + "short_label" : "Final", + "weight" : 0.4 + } + ], + "GRADE_CUTOFFS" : { + "A" : 0.87, + "B" : 0.7, + "C" : 0.6 + } + } + """ + + # Load the global settings as a dictionary + grading_policy = json.loads(default_policy_string) + + # Load the course policies as a dictionary + course_policy = {} + if course_policy_string: + course_policy = json.loads(course_policy_string) + + # Override any global settings with the course settings + grading_policy.update(course_policy) + + # Here is where we should parse any configurations, so that we can fail early + grading_policy['GRADER'] = grader_from_conf(grading_policy['GRADER']) + + return grading_policy + def aggregate_scores(scores, section_name="summary"): """ diff --git a/common/lib/xmodule/xmodule/html_checker.py b/common/lib/xmodule/xmodule/html_checker.py new file mode 100644 index 0000000000..5e6b417d28 --- /dev/null +++ b/common/lib/xmodule/xmodule/html_checker.py @@ -0,0 +1,14 @@ +from lxml import etree + +def check_html(html): + ''' + Check whether the passed in html string can be parsed by lxml. + Return bool success. + ''' + parser = etree.HTMLParser() + try: + etree.fromstring(html, parser) + return True + except Exception as err: + pass + return False diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index b9bc34aed6..260b84278b 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -1,13 +1,18 @@ +import copy +from fs.errors import ResourceNotFoundError import logging import os +import sys from lxml import etree -from xmodule.x_module import XModule -from xmodule.raw_module import RawDescriptor +from .x_module import XModule +from .xml_module import XmlDescriptor +from .editing_module import EditingDescriptor +from .stringify import stringify_children +from .html_checker import check_html log = logging.getLogger("mitx.courseware") - class HtmlModule(XModule): def get_html(self): return self.html @@ -19,33 +24,118 @@ class HtmlModule(XModule): self.html = self.definition['data'] -class HtmlDescriptor(RawDescriptor): +class HtmlDescriptor(XmlDescriptor, EditingDescriptor): """ Module for putting raw html in a course """ mako_template = "widgets/html-edit.html" module_class = HtmlModule - filename_extension = "html" + filename_extension = "xml" - # TODO (cpennington): Delete this method once all fall 2012 course are being - # edited in the cms + # VS[compat] TODO (cpennington): Delete this method once all fall 2012 course + # are being edited in the cms @classmethod def backcompat_paths(cls, path): - if path.endswith('.html.html'): - path = path[:-5] + origpath = path + if path.endswith('.html.xml'): + path = path[:-9] + '.html' #backcompat--look for html instead of xml candidates = [] while os.sep in path: candidates.append(path) _, _, path = path.partition(os.sep) + # also look for .html versions instead of .xml + if origpath.endswith('.xml'): + candidates.append(origpath[:-4] + '.html') return candidates + # NOTE: html descriptors are special. We do not want to parse and + # export them ourselves, because that can break things (e.g. lxml + # adds body tags when it exports, but they should just be html + # snippets that will be included in the middle of pages. + @classmethod - def file_to_xml(cls, file_object): - parser = etree.HTMLParser() - return etree.parse(file_object, parser).getroot() + def load_definition(cls, xml_object, system, location): + '''Load a descriptor from the specified xml_object: + + If there is a filename attribute, load it as a string, and + log a warning if it is not parseable by etree.HTMLParser. + + If there is not a filename attribute, the definition is the body + of the xml_object, without the root tag (do not want in the + middle of a page) + ''' + filename = xml_object.get('filename') + if filename is None: + definition_xml = copy.deepcopy(xml_object) + cls.clean_metadata_from_xml(definition_xml) + return {'data' : stringify_children(definition_xml)} + else: + filepath = cls._format_filepath(xml_object.tag, filename) + + # VS[compat] + # TODO (cpennington): If the file doesn't exist at the right path, + # give the class a chance to fix it up. The file will be written out + # again in the correct format. This should go away once the CMS is + # online and has imported all current (fall 2012) courses from xml + if not system.resources_fs.exists(filepath): + candidates = cls.backcompat_paths(filepath) + #log.debug("candidates = {0}".format(candidates)) + for candidate in candidates: + if system.resources_fs.exists(candidate): + filepath = candidate + break + + try: + with system.resources_fs.open(filepath) as file: + html = file.read() + # Log a warning if we can't parse the file, but don't error + if not check_html(html): + msg = "Couldn't parse html in {0}.".format(filepath) + log.warning(msg) + system.error_tracker("Warning: " + msg) + + definition = {'data' : html} + + # TODO (ichuang): remove this after migration + # for Fall 2012 LMS migration: keep filename (and unmangled filename) + definition['filename'] = [ filepath, filename ] + + return definition + + except (ResourceNotFoundError) as err: + msg = 'Unable to load file contents at path {0}: {1} '.format( + filepath, err) + # add more info and re-raise + raise Exception(msg), None, sys.exc_info()[2] @classmethod def split_to_file(cls, xml_object): - # never include inline html + '''Never include inline html''' return True + + + # TODO (vshnayder): make export put things in the right places. + + def definition_to_xml(self, resource_fs): + '''If the contents are valid xml, write them to filename.xml. Otherwise, + write just the tag to filename.xml, and the html + string to filename.html. + ''' + try: + return etree.fromstring(self.definition['data']) + except etree.XMLSyntaxError: + pass + + # Not proper format. Write html to file, return an empty tag + filepath = u'{category}/{name}.html'.format(category=self.category, + name=self.url_name) + + resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True) + with resource_fs.open(filepath, 'w') as file: + file.write(self.definition['data']) + + elt = etree.Element('html') + elt.set("filename", self.url_name) + return elt + diff --git a/common/lib/xmodule/xmodule/mako_module.py b/common/lib/xmodule/xmodule/mako_module.py index fcc47aaaaf..eedac99aa8 100644 --- a/common/lib/xmodule/xmodule/mako_module.py +++ b/common/lib/xmodule/xmodule/mako_module.py @@ -2,10 +2,10 @@ from x_module import XModuleDescriptor, DescriptorSystem class MakoDescriptorSystem(DescriptorSystem): - def __init__(self, load_item, resources_fs, error_handler, - render_template): + def __init__(self, load_item, resources_fs, error_tracker, + render_template, **kwargs): super(MakoDescriptorSystem, self).__init__( - load_item, resources_fs, error_handler) + load_item, resources_fs, error_tracker, **kwargs) self.render_template = render_template diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index 279782b61a..e59e4bd68e 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -3,10 +3,13 @@ This module provides an abstraction for working with XModuleDescriptors that are stored in a database an accessible using their Location as an identifier """ -import re -from collections import namedtuple -from .exceptions import InvalidLocationError import logging +import re + +from collections import namedtuple + +from .exceptions import InvalidLocationError, InsufficientSpecificationError +from xmodule.errortracker import ErrorLog, make_error_tracker log = logging.getLogger('mitx.' + 'modulestore') @@ -38,15 +41,15 @@ class Location(_LocationBase): ''' __slots__ = () - @classmethod - def clean(cls, value): + @staticmethod + def clean(value): """ Return value, made into a form legal for locations """ return re.sub('_+', '_', INVALID_CHARS.sub('_', value)) - @classmethod - def is_valid(cls, value): + @staticmethod + def is_valid(value): ''' Check if the value is a valid location, in any acceptable format. ''' @@ -56,6 +59,21 @@ class Location(_LocationBase): return False return True + @staticmethod + def ensure_fully_specified(location): + '''Make sure location is valid, and fully specified. Raises + InvalidLocationError or InsufficientSpecificationError if not. + + returns a Location object corresponding to location. + ''' + loc = Location(location) + for key, val in loc.dict().iteritems(): + if key != 'revision' and val is None: + raise InsufficientSpecificationError(location) + return loc + + + def __new__(_cls, loc_or_tag=None, org=None, course=None, category=None, name=None, revision=None): """ @@ -198,6 +216,18 @@ class ModuleStore(object): """ raise NotImplementedError + def get_item_errors(self, location): + """ + Return a list of (msg, exception-or-None) errors that the modulestore + encountered when loading the item at location. + + location : something that can be passed to Location + + Raises the same exceptions as get_item if the location isn't found or + isn't fully specified. + """ + raise NotImplementedError + def get_items(self, location, depth=0): """ Returns a list of XModuleDescriptor instances for the items @@ -254,25 +284,47 @@ class ModuleStore(object): ''' raise NotImplementedError - def path_to_location(self, location, course=None, chapter=None, section=None): - ''' - Try to find a course/chapter/section[/position] path to this location. - raise ItemNotFoundError if the location doesn't exist. + def get_parent_locations(self, location): + '''Find all locations that are the parents of this location. Needed + for path_to_location(). - If course, chapter, section are not None, restrict search to paths with those - components as specified. - - raise NoPathToItem if the location exists, but isn't accessible via - a path that matches the course/chapter/section restrictions. - - In general, a location may be accessible via many paths. This method may - return any valid path. - - Return a tuple (course, chapter, section, position). - - If the section a sequence, position should be the position of this location - in that sequence. Otherwise, position should be None. + returns an iterable of things that can be passed to Location. ''' raise NotImplementedError + +class ModuleStoreBase(ModuleStore): + ''' + Implement interface functionality that can be shared. + ''' + def __init__(self): + ''' + Set up the error-tracking logic. + ''' + self._location_errors = {} # location -> ErrorLog + + def _get_errorlog(self, location): + """ + If we already have an errorlog for this location, return it. Otherwise, + create one. + """ + location = Location(location) + if location not in self._location_errors: + self._location_errors[location] = make_error_tracker() + return self._location_errors[location] + + def get_item_errors(self, location): + """ + Return list of errors for this location, if any. Raise the same + errors as get_item if location isn't present. + + NOTE: For now, the only items that track errors are CourseDescriptors in + the xml datastore. This will return an empty list for all other items + and datastores. + """ + # check that item is present and raise the promised exceptions if needed + self.get_item(location) + + errorlog = self._get_errorlog(location) + return errorlog.errors diff --git a/common/lib/xmodule/xmodule/modulestore/mongo.py b/common/lib/xmodule/xmodule/modulestore/mongo.py index df4e20f3a7..1cec6c7f87 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo.py @@ -6,14 +6,13 @@ from itertools import repeat from path import path from importlib import import_module -from xmodule.errorhandlers import strict_error_handler +from xmodule.errortracker import null_error_tracker from xmodule.x_module import XModuleDescriptor from xmodule.mako_module import MakoDescriptorSystem -from xmodule.course_module import CourseDescriptor from mitxmako.shortcuts import render_to_string -from . import ModuleStore, Location -from .exceptions import (ItemNotFoundError, InsufficientSpecificationError, +from . import ModuleStoreBase, Location +from .exceptions import (ItemNotFoundError, NoPathToItem, DuplicateItemError) # TODO (cpennington): This code currently operates under the assumption that @@ -27,7 +26,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem): from, with a backup of calling to the underlying modulestore for more data """ def __init__(self, modulestore, module_data, default_class, resources_fs, - error_handler, render_template): + error_tracker, render_template): """ modulestore: the module store that can be used to retrieve additional modules @@ -39,13 +38,13 @@ class CachingDescriptorSystem(MakoDescriptorSystem): resources_fs: a filesystem, as per MakoDescriptorSystem - error_handler: + error_tracker: a function that logs errors for later display to users render_template: a function for rendering templates, as per MakoDescriptorSystem """ super(CachingDescriptorSystem, self).__init__( - self.load_item, resources_fs, error_handler, render_template) + self.load_item, resources_fs, error_tracker, render_template) self.modulestore = modulestore self.module_data = module_data self.default_class = default_class @@ -74,13 +73,17 @@ def location_to_query(location): return query -class MongoModuleStore(ModuleStore): +class MongoModuleStore(ModuleStoreBase): """ A Mongodb backed ModuleStore """ # TODO (cpennington): Enable non-filesystem filestores - def __init__(self, host, db, collection, fs_root, port=27017, default_class=None): + def __init__(self, host, db, collection, fs_root, port=27017, default_class=None, + error_tracker=null_error_tracker): + + ModuleStoreBase.__init__(self) + self.collection = pymongo.connection.Connection( host=host, port=port @@ -91,13 +94,17 @@ class MongoModuleStore(ModuleStore): # Force mongo to maintain an index over _id.* that is in the same order # that is used when querying by a location - self.collection.ensure_index(zip(('_id.' + field for field in Location._fields), repeat(1))) + self.collection.ensure_index( + zip(('_id.' + field for field in Location._fields), repeat(1))) - # TODO (vshnayder): default arg default_class=None will make this error - module_path, _, class_name = default_class.rpartition('.') - class_ = getattr(import_module(module_path), class_name) - self.default_class = class_ + if default_class is not None: + module_path, _, class_name = default_class.rpartition('.') + class_ = getattr(import_module(module_path), class_name) + self.default_class = class_ + else: + self.default_class = None self.fs_root = path(fs_root) + self.error_tracker = error_tracker def _clean_item_data(self, item): """ @@ -149,7 +156,7 @@ class MongoModuleStore(ModuleStore): data_cache, self.default_class, resource_fs, - strict_error_handler, + self.error_tracker, render_to_string, ) return system.load_item(item['location']) @@ -172,12 +179,17 @@ class MongoModuleStore(ModuleStore): return self.get_items(course_filter) def _find_one(self, location): - '''Look for a given location in the collection. - If revision isn't specified, returns the latest.''' - return self.collection.find_one( + '''Look for a given location in the collection. If revision is not + specified, returns the latest. If the item is not present, raise + ItemNotFoundError. + ''' + item = self.collection.find_one( location_to_query(location), sort=[('revision', pymongo.ASCENDING)], ) + if item is None: + raise ItemNotFoundError(location) + return item def get_item(self, location, depth=0): """ @@ -197,14 +209,8 @@ class MongoModuleStore(ModuleStore): calls to get_children() to cache. None indicates to cache all descendents. """ - - for key, val in Location(location).dict().iteritems(): - if key != 'revision' and val is None: - raise InsufficientSpecificationError(location) - + location = Location.ensure_fully_specified(location) item = self._find_one(location) - if item is None: - raise ItemNotFoundError(location) return self._load_items([item], depth)[0] def get_items(self, location, depth=0): @@ -282,96 +288,20 @@ class MongoModuleStore(ModuleStore): ) def get_parent_locations(self, location): - '''Find all locations that are the parents of this location. - Mostly intended for use in path_to_location, but exposed for testing - and possible other usefulness. + '''Find all locations that are the parents of this location. Needed + for path_to_location(). - returns an iterable of things that can be passed to Location. + If there is no data at location in this modulestore, raise + ItemNotFoundError. + + returns an iterable of things that can be passed to Location. This may + be empty if there are no parents. ''' - location = Location(location) - items = self.collection.find({'definition.children': str(location)}, + location = Location.ensure_fully_specified(location) + # Check that it's actually in this modulestore. + item = self._find_one(location) + # now get the parents + items = self.collection.find({'definition.children': location.url()}, {'_id': True}) return [i['_id'] for i in items] - def path_to_location(self, location, course_name=None): - ''' - Try to find a course_id/chapter/section[/position] path to this location. - The courseware insists that the first level in the course is chapter, - but any kind of module can be a "section". - - location: something that can be passed to Location - course_name: [optional]. If not None, restrict search to paths - in that course. - - raise ItemNotFoundError if the location doesn't exist. - - raise NoPathToItem if the location exists, but isn't accessible via - a chapter/section path in the course(s) being searched. - - Return a tuple (course_id, chapter, section, position) suitable for the - courseware index view. - - A location may be accessible via many paths. This method may - return any valid path. - - If the section is a sequence, position will be the position - of this location in that sequence. Otherwise, position will - be None. TODO (vshnayder): Not true yet. - ''' - # Check that location is present at all - if self._find_one(location) is None: - raise ItemNotFoundError(location) - - def flatten(xs): - '''Convert lisp-style (a, (b, (c, ()))) lists into a python list. - Not a general flatten function. ''' - p = [] - while xs != (): - p.append(xs[0]) - xs = xs[1] - return p - - def find_path_to_course(location, course_name=None): - '''Find a path up the location graph to a node with the - specified category. If no path exists, return None. If a - path exists, return it as a list with target location - first, and the starting location last. - ''' - # Standard DFS - - # To keep track of where we came from, the work queue has - # tuples (location, path-so-far). To avoid lots of - # copying, the path-so-far is stored as a lisp-style - # list--nested hd::tl tuples, and flattened at the end. - queue = [(location, ())] - while len(queue) > 0: - (loc, path) = queue.pop() # Takes from the end - loc = Location(loc) - # print 'Processing loc={0}, path={1}'.format(loc, path) - if loc.category == "course": - if course_name is None or course_name == loc.name: - # Found it! - path = (loc, path) - return flatten(path) - - # otherwise, add parent locations at the end - newpath = (loc, path) - parents = self.get_parent_locations(loc) - queue.extend(zip(parents, repeat(newpath))) - - # If we're here, there is no path - return None - - path = find_path_to_course(location, course_name) - if path is None: - raise(NoPathToItem(location)) - - n = len(path) - course_id = CourseDescriptor.location_to_id(path[0]) - chapter = path[1].name if n > 1 else None - section = path[2].name if n > 2 else None - - # TODO (vshnayder): not handling position at all yet... - position = None - - return (course_id, chapter, section, position) diff --git a/common/lib/xmodule/xmodule/modulestore/search.py b/common/lib/xmodule/xmodule/modulestore/search.py new file mode 100644 index 0000000000..baf3d46b57 --- /dev/null +++ b/common/lib/xmodule/xmodule/modulestore/search.py @@ -0,0 +1,97 @@ +from itertools import repeat + +from xmodule.course_module import CourseDescriptor + +from .exceptions import (ItemNotFoundError, NoPathToItem) +from . import ModuleStore, Location + + +def path_to_location(modulestore, location, course_name=None): + ''' + Try to find a course_id/chapter/section[/position] path to location in + modulestore. The courseware insists that the first level in the course is + chapter, but any kind of module can be a "section". + + location: something that can be passed to Location + course_name: [optional]. If not None, restrict search to paths + in that course. + + raise ItemNotFoundError if the location doesn't exist. + + raise NoPathToItem if the location exists, but isn't accessible via + a chapter/section path in the course(s) being searched. + + Return a tuple (course_id, chapter, section, position) suitable for the + courseware index view. + + A location may be accessible via many paths. This method may + return any valid path. + + If the section is a sequence, position will be the position + of this location in that sequence. Otherwise, position will + be None. TODO (vshnayder): Not true yet. + ''' + + def flatten(xs): + '''Convert lisp-style (a, (b, (c, ()))) list into a python list. + Not a general flatten function. ''' + p = [] + while xs != (): + p.append(xs[0]) + xs = xs[1] + return p + + def find_path_to_course(location, course_name=None): + '''Find a path up the location graph to a node with the + specified category. + + If no path exists, return None. + + If a path exists, return it as a list with target location first, and + the starting location last. + ''' + # Standard DFS + + # To keep track of where we came from, the work queue has + # tuples (location, path-so-far). To avoid lots of + # copying, the path-so-far is stored as a lisp-style + # list--nested hd::tl tuples, and flattened at the end. + queue = [(location, ())] + while len(queue) > 0: + (loc, path) = queue.pop() # Takes from the end + loc = Location(loc) + + # get_parent_locations should raise ItemNotFoundError if location + # isn't found so we don't have to do it explicitly. Call this + # first to make sure the location is there (even if it's a course, and + # we would otherwise immediately exit). + parents = modulestore.get_parent_locations(loc) + + # print 'Processing loc={0}, path={1}'.format(loc, path) + if loc.category == "course": + if course_name is None or course_name == loc.name: + # Found it! + path = (loc, path) + return flatten(path) + + # otherwise, add parent locations at the end + newpath = (loc, path) + queue.extend(zip(parents, repeat(newpath))) + + # If we're here, there is no path + return None + + path = find_path_to_course(location, course_name) + if path is None: + raise(NoPathToItem(location)) + + n = len(path) + course_id = CourseDescriptor.location_to_id(path[0]) + # pull out the location names + chapter = path[1].name if n > 1 else None + section = path[2].name if n > 2 else None + + # TODO (vshnayder): not handling position at all yet... + position = None + + return (course_id, chapter, section, position) diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py index cb2bc6e20c..24f0441ee0 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py @@ -8,6 +8,7 @@ from xmodule.modulestore import Location from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, NoPathToItem from xmodule.modulestore.mongo import MongoModuleStore from xmodule.modulestore.xml_importer import import_from_xml +from xmodule.modulestore.search import path_to_location # from ~/mitx_all/mitx/common/lib/xmodule/xmodule/modulestore/tests/ # to ~/mitx_all/mitx/common/test @@ -28,7 +29,7 @@ DEFAULT_CLASS = 'xmodule.raw_module.RawDescriptor' class TestMongoModuleStore(object): - + '''Tests!''' @classmethod def setupClass(cls): cls.connection = pymongo.connection.Connection(HOST, PORT) @@ -67,7 +68,7 @@ class TestMongoModuleStore(object): def test_init(self): '''Make sure the db loads, and print all the locations in the db. - Call this directly from failing tests to see what's loaded''' + Call this directly from failing tests to see what is loaded''' ids = list(self.connection[DB][COLLECTION].find({}, {'_id': True})) pprint([Location(i['_id']).url() for i in ids]) @@ -93,8 +94,6 @@ class TestMongoModuleStore(object): self.store.get_item("i4x://edX/toy/video/Welcome"), None) - - def test_find_one(self): assert_not_equals( self.store._find_one(Location("i4x://edX/toy/course/2012_Fall")), @@ -117,13 +116,13 @@ class TestMongoModuleStore(object): ("edX/toy/2012_Fall", "Overview", "Toy_Videos", None)), ) for location, expected in should_work: - assert_equals(self.store.path_to_location(location), expected) + assert_equals(path_to_location(self.store, location), expected) not_found = ( - "i4x://edX/toy/video/WelcomeX", + "i4x://edX/toy/video/WelcomeX", "i4x://edX/toy/course/NotHome" ) for location in not_found: - assert_raises(ItemNotFoundError, self.store.path_to_location, location) + assert_raises(ItemNotFoundError, path_to_location, self.store, location) # Since our test files are valid, there shouldn't be any # elements with no path to them. But we can look for them in @@ -132,5 +131,5 @@ class TestMongoModuleStore(object): "i4x://edX/simple/video/Lost_Video", ) for location in no_path: - assert_raises(NoPathToItem, self.store.path_to_location, location, "toy") + assert_raises(NoPathToItem, path_to_location, self.store, location, "toy") diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py index 7dd6868f78..46fcf19469 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml.py +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -1,16 +1,18 @@ import logging +import os +import re + from fs.osfs import OSFS from importlib import import_module from lxml import etree from path import path -from xmodule.errorhandlers import logging_error_handler +from xmodule.errortracker import ErrorLog, make_error_tracker from xmodule.x_module import XModuleDescriptor, XMLParsingSystem +from xmodule.course_module import CourseDescriptor from xmodule.mako_module import MakoDescriptorSystem from cStringIO import StringIO -import os -import re -from . import ModuleStore, Location +from . import ModuleStoreBase, Location from .exceptions import ItemNotFoundError etree.set_default_parser( @@ -19,7 +21,6 @@ etree.set_default_parser( log = logging.getLogger('mitx.' + __name__) - # VS[compat] # TODO (cpennington): Remove this once all fall 2012 courses have been imported # into the cms from xml @@ -29,7 +30,7 @@ def clean_out_mako_templating(xml_string): return xml_string class ImportSystem(XMLParsingSystem, MakoDescriptorSystem): - def __init__(self, xmlstore, org, course, course_dir, error_handler): + def __init__(self, xmlstore, org, course, course_dir, error_tracker, **kwargs): """ A class that handles loading from xml. Does some munging to ensure that all elements have unique slugs. @@ -40,6 +41,9 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem): self.used_slugs = set() def process_xml(xml): + """Takes an xml string, and returns a XModuleDescriptor created from + that xml. + """ try: # VS[compat] # TODO (cpennington): Remove this once all fall 2012 courses @@ -70,37 +74,36 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem): # log.debug('-> slug=%s' % slug) xml_data.set('url_name', slug) - module = XModuleDescriptor.load_from_xml( + descriptor = XModuleDescriptor.load_from_xml( etree.tostring(xml_data), self, org, course, xmlstore.default_class) - #log.debug('==> importing module location %s' % repr(module.location)) - module.metadata['data_dir'] = course_dir + #log.debug('==> importing descriptor location %s' % + # repr(descriptor.location)) + descriptor.metadata['data_dir'] = course_dir - xmlstore.modules[module.location] = module + xmlstore.modules[descriptor.location] = descriptor if xmlstore.eager: - module.get_children() - return module + descriptor.get_children() + return descriptor render_template = lambda: '' load_item = xmlstore.get_item resources_fs = OSFS(xmlstore.data_dir / course_dir) MakoDescriptorSystem.__init__(self, load_item, resources_fs, - error_handler, render_template) + error_tracker, render_template, **kwargs) XMLParsingSystem.__init__(self, load_item, resources_fs, - error_handler, process_xml) + error_tracker, process_xml, **kwargs) - -class XMLModuleStore(ModuleStore): +class XMLModuleStore(ModuleStoreBase): """ An XML backed ModuleStore """ def __init__(self, data_dir, default_class=None, eager=False, - course_dirs=None, - error_handler=logging_error_handler): + course_dirs=None): """ Initialize an XMLModuleStore from data_dir @@ -114,17 +117,13 @@ class XMLModuleStore(ModuleStore): course_dirs: If specified, the list of course_dirs to load. Otherwise, load all course dirs - - error_handler: The error handler used here and in the underlying - DescriptorSystem. By default, raise exceptions for all errors. - See the comments in x_module.py:DescriptorSystem """ + ModuleStoreBase.__init__(self) self.eager = eager self.data_dir = path(data_dir) self.modules = {} # location -> XModuleDescriptor self.courses = {} # course_dir -> XModuleDescriptor for the course - self.error_handler = error_handler if default_class is None: self.default_class = None @@ -147,16 +146,32 @@ class XMLModuleStore(ModuleStore): os.path.exists(self.data_dir / d / "course.xml")] for course_dir in course_dirs: - try: - course_descriptor = self.load_course(course_dir) - self.courses[course_dir] = course_descriptor - except: - msg = "Failed to load course '%s'" % course_dir - log.exception(msg) - error_handler(msg) + self.try_load_course(course_dir) + def try_load_course(self,course_dir): + ''' + Load a course, keeping track of errors as we go along. + ''' + try: + # Special-case code here, since we don't have a location for the + # course before it loads. + # So, make a tracker to track load-time errors, then put in the right + # place after the course loads and we have its location + errorlog = make_error_tracker() + course_descriptor = self.load_course(course_dir, errorlog.tracker) + self.courses[course_dir] = course_descriptor + self._location_errors[course_descriptor.location] = errorlog + except: + msg = "Failed to load course '%s'" % course_dir + log.exception(msg) - def load_course(self, course_dir): + def __unicode__(self): + ''' + String representation - for debugging + ''' + return 'data_dir=%s, %d courses, %d modules' % (self.data_dir,len(self.courses),len(self.modules)) + + def load_course(self, course_dir, tracker): """ Load a course into this module store course_path: Course directory name @@ -190,13 +205,13 @@ class XMLModuleStore(ModuleStore): )) course = course_dir - system = ImportSystem(self, org, course, course_dir, - self.error_handler) + system = ImportSystem(self, org, course, course_dir, tracker) course_descriptor = system.process_xml(etree.tostring(course_data)) log.debug('========> Done with course import from {0}'.format(course_dir)) return course_descriptor + def get_item(self, location, depth=0): """ Returns an XModuleDescriptor instance for the item at location. @@ -217,15 +232,19 @@ class XMLModuleStore(ModuleStore): except KeyError: raise ItemNotFoundError(location) + def get_courses(self, depth=0): """ - Returns a list of course descriptors + Returns a list of course descriptors. If there were errors on loading, + some of these may be ErrorDescriptors instead. """ return self.courses.values() + def create_item(self, location): raise NotImplementedError("XMLModuleStores are read-only") + def update_item(self, location, data): """ Set the data in the item specified by the location to @@ -236,6 +255,7 @@ class XMLModuleStore(ModuleStore): """ raise NotImplementedError("XMLModuleStores are read-only") + def update_children(self, location, children): """ Set the children for the item specified by the location to @@ -246,6 +266,7 @@ class XMLModuleStore(ModuleStore): """ raise NotImplementedError("XMLModuleStores are read-only") + def update_metadata(self, location, metadata): """ Set the metadata for the item specified by the location to diff --git a/common/lib/xmodule/xmodule/modulestore/xml_importer.py b/common/lib/xmodule/xmodule/modulestore/xml_importer.py index 578ade95fe..891db7e994 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml_importer.py +++ b/common/lib/xmodule/xmodule/modulestore/xml_importer.py @@ -35,6 +35,8 @@ def import_from_xml(store, data_dir, course_dirs=None, eager=True, store.update_item(module.location, module.definition['data']) if 'children' in module.definition: store.update_children(module.location, module.definition['children']) - store.update_metadata(module.location, dict(module.metadata)) + # NOTE: It's important to use own_metadata here to avoid writing + # inherited metadata everywhere. + store.update_metadata(module.location, dict(module.own_metadata)) return module_store diff --git a/common/lib/xmodule/xmodule/raw_module.py b/common/lib/xmodule/xmodule/raw_module.py index 90f4139bd5..9fdb5d0b38 100644 --- a/common/lib/xmodule/xmodule/raw_module.py +++ b/common/lib/xmodule/xmodule/raw_module.py @@ -1,27 +1,16 @@ -from pkg_resources import resource_string from lxml import etree -from xmodule.mako_module import MakoModuleDescriptor +from xmodule.editing_module import EditingDescriptor from xmodule.xml_module import XmlDescriptor import logging +import sys log = logging.getLogger(__name__) - -class RawDescriptor(MakoModuleDescriptor, XmlDescriptor): +class RawDescriptor(XmlDescriptor, EditingDescriptor): """ - Module that provides a raw editing view of its data and children + Module that provides a raw editing view of its data and children. It + requires that the definition xml is valid. """ - mako_template = "widgets/raw-edit.html" - - js = {'coffee': [resource_string(__name__, 'js/src/raw/edit.coffee')]} - js_module_name = "RawDescriptor" - - def get_context(self): - return { - 'module': self, - 'data': self.definition['data'], - } - @classmethod def definition_from_xml(cls, xml_object, system): return {'data': etree.tostring(xml_object)} @@ -30,13 +19,12 @@ class RawDescriptor(MakoModuleDescriptor, XmlDescriptor): try: return etree.fromstring(self.definition['data']) except etree.XMLSyntaxError as err: + # Can't recover here, so just add some info and + # re-raise lines = self.definition['data'].split('\n') line, offset = err.position msg = ("Unable to create xml for problem {loc}. " "Context: '{context}'".format( context=lines[line - 1][offset - 40:offset + 40], loc=self.location)) - log.exception(msg) - self.system.error_handler(msg) - # no workaround possible, so just re-raise - raise + raise Exception, msg, sys.exc_info()[2] diff --git a/common/lib/xmodule/xmodule/stringify.py b/common/lib/xmodule/xmodule/stringify.py new file mode 100644 index 0000000000..dad964140f --- /dev/null +++ b/common/lib/xmodule/xmodule/stringify.py @@ -0,0 +1,20 @@ +from itertools import chain +from lxml import etree + +def stringify_children(node): + ''' + Return all contents of an xml tree, without the outside tags. + e.g. if node is parse of + "Hi
      there Bruce!
      " + should return + "Hi
      there Bruce!
      " + + fixed from + http://stackoverflow.com/questions/4624062/get-all-text-inside-a-tag-in-lxml + ''' + parts = ([node.text] + + list(chain(*([etree.tostring(c), c.tail] + for c in node.getchildren()) + ))) + # filter removes possible Nones in texts and tails + return ''.join(filter(None, parts)) diff --git a/common/lib/xmodule/xmodule/template_module.py b/common/lib/xmodule/xmodule/template_module.py index 3f926555f4..260ad009f9 100644 --- a/common/lib/xmodule/xmodule/template_module.py +++ b/common/lib/xmodule/xmodule/template_module.py @@ -7,16 +7,14 @@ from mako.template import Template class CustomTagModule(XModule): """ This module supports tags of the form - - $tagname - + In this case, $tagname should refer to a file in data/custom_tags, which contains a mako template that uses ${option} and ${option2} for the content. For instance: - data/custom_tags/book:: + data/mycourse/custom_tags/book:: More information given in the text course.xml:: @@ -34,7 +32,18 @@ class CustomTagModule(XModule): instance_state, shared_state, **kwargs) xmltree = etree.fromstring(self.definition['data']) - template_name = xmltree.attrib['impl'] + if 'impl' in xmltree.attrib: + template_name = xmltree.attrib['impl'] + else: + # VS[compat] backwards compatibility with old nested customtag structure + child_impl = xmltree.find('impl') + if child_impl is not None: + template_name = child_impl.text + else: + # TODO (vshnayder): better exception type + raise Exception("Could not find impl attribute in customtag {0}" + .format(location)) + params = dict(xmltree.items()) with self.system.filestore.open( 'custom_tags/{name}'.format(name=template_name)) as template: diff --git a/common/lib/xmodule/tests/__init__.py b/common/lib/xmodule/xmodule/tests/__init__.py similarity index 99% rename from common/lib/xmodule/tests/__init__.py rename to common/lib/xmodule/xmodule/tests/__init__.py index 94bf1da65e..7f6fcfe00c 100644 --- a/common/lib/xmodule/tests/__init__.py +++ b/common/lib/xmodule/xmodule/tests/__init__.py @@ -31,7 +31,8 @@ i4xs = ModuleSystem( user=Mock(), filestore=fs.osfs.OSFS(os.path.dirname(os.path.realpath(__file__))), debug=True, - xqueue_callback_url='/' + xqueue_callback_url='/', + is_staff=False ) diff --git a/common/lib/xmodule/tests/test_export.py b/common/lib/xmodule/xmodule/tests/test_export.py similarity index 85% rename from common/lib/xmodule/tests/test_export.py rename to common/lib/xmodule/xmodule/tests/test_export.py index 97da2c4fe5..eacf8352be 100644 --- a/common/lib/xmodule/tests/test_export.py +++ b/common/lib/xmodule/xmodule/tests/test_export.py @@ -1,5 +1,6 @@ from xmodule.modulestore.xml import XMLModuleStore from nose.tools import assert_equals +from nose import SkipTest from tempfile import mkdtemp from fs.osfs import OSFS @@ -26,3 +27,10 @@ def check_export_roundtrip(data_dir): for location in initial_import.modules.keys(): print "Checking", location assert_equals(initial_import.modules[location], second_import.modules[location]) + + +def test_toy_roundtrip(): + dir = "" + # TODO: add paths and make this run. + raise SkipTest() + check_export_roundtrip(dir) diff --git a/common/lib/xmodule/tests/test_files/choiceresponse_checkbox.xml b/common/lib/xmodule/xmodule/tests/test_files/choiceresponse_checkbox.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/choiceresponse_checkbox.xml rename to common/lib/xmodule/xmodule/tests/test_files/choiceresponse_checkbox.xml diff --git a/common/lib/xmodule/tests/test_files/choiceresponse_radio.xml b/common/lib/xmodule/xmodule/tests/test_files/choiceresponse_radio.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/choiceresponse_radio.xml rename to common/lib/xmodule/xmodule/tests/test_files/choiceresponse_radio.xml diff --git a/common/lib/xmodule/tests/test_files/coderesponse.xml b/common/lib/xmodule/xmodule/tests/test_files/coderesponse.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/coderesponse.xml rename to common/lib/xmodule/xmodule/tests/test_files/coderesponse.xml diff --git a/common/lib/xmodule/tests/test_files/formularesponse_with_hint.xml b/common/lib/xmodule/xmodule/tests/test_files/formularesponse_with_hint.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/formularesponse_with_hint.xml rename to common/lib/xmodule/xmodule/tests/test_files/formularesponse_with_hint.xml diff --git a/common/lib/xmodule/tests/test_files/imageresponse.xml b/common/lib/xmodule/xmodule/tests/test_files/imageresponse.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/imageresponse.xml rename to common/lib/xmodule/xmodule/tests/test_files/imageresponse.xml diff --git a/common/lib/xmodule/tests/test_files/multi_bare.xml b/common/lib/xmodule/xmodule/tests/test_files/multi_bare.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/multi_bare.xml rename to common/lib/xmodule/xmodule/tests/test_files/multi_bare.xml diff --git a/common/lib/xmodule/tests/test_files/multichoice.xml b/common/lib/xmodule/xmodule/tests/test_files/multichoice.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/multichoice.xml rename to common/lib/xmodule/xmodule/tests/test_files/multichoice.xml diff --git a/common/lib/xmodule/tests/test_files/optionresponse.xml b/common/lib/xmodule/xmodule/tests/test_files/optionresponse.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/optionresponse.xml rename to common/lib/xmodule/xmodule/tests/test_files/optionresponse.xml diff --git a/common/lib/xmodule/tests/test_files/stringresponse_with_hint.xml b/common/lib/xmodule/xmodule/tests/test_files/stringresponse_with_hint.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/stringresponse_with_hint.xml rename to common/lib/xmodule/xmodule/tests/test_files/stringresponse_with_hint.xml diff --git a/common/lib/xmodule/tests/test_files/symbolicresponse.xml b/common/lib/xmodule/xmodule/tests/test_files/symbolicresponse.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/symbolicresponse.xml rename to common/lib/xmodule/xmodule/tests/test_files/symbolicresponse.xml diff --git a/common/lib/xmodule/tests/test_files/truefalse.xml b/common/lib/xmodule/xmodule/tests/test_files/truefalse.xml similarity index 100% rename from common/lib/xmodule/tests/test_files/truefalse.xml rename to common/lib/xmodule/xmodule/tests/test_files/truefalse.xml diff --git a/common/lib/xmodule/xmodule/tests/test_import.py b/common/lib/xmodule/xmodule/tests/test_import.py new file mode 100644 index 0000000000..0d3e2260fb --- /dev/null +++ b/common/lib/xmodule/xmodule/tests/test_import.py @@ -0,0 +1,140 @@ +from path import path +import unittest +from fs.memoryfs import MemoryFS + +from lxml import etree + +from xmodule.x_module import XMLParsingSystem, XModuleDescriptor +from xmodule.errortracker import make_error_tracker +from xmodule.modulestore import Location +from xmodule.modulestore.exceptions import ItemNotFoundError + +ORG = 'test_org' +COURSE = 'test_course' + +class DummySystem(XMLParsingSystem): + def __init__(self): + + self.modules = {} + self.resources_fs = MemoryFS() + self.errorlog = make_error_tracker() + + def load_item(loc): + loc = Location(loc) + if loc in self.modules: + return self.modules[loc] + + print "modules: " + print self.modules + raise ItemNotFoundError("Can't find item at loc: {0}".format(loc)) + + def process_xml(xml): + print "loading {0}".format(xml) + descriptor = XModuleDescriptor.load_from_xml(xml, self, ORG, COURSE, None) + # Need to save module so we can find it later + self.modules[descriptor.location] = descriptor + + # always eager + descriptor.get_children() + return descriptor + + + XMLParsingSystem.__init__(self, load_item, self.resources_fs, + self.errorlog.tracker, process_xml) + + def render_template(self, template, context): + raise Exception("Shouldn't be called") + + + + +class ImportTestCase(unittest.TestCase): + '''Make sure module imports work properly, including for malformed inputs''' + + + @staticmethod + def get_system(): + '''Get a dummy system''' + return DummySystem() + + def test_fallback(self): + '''Make sure that malformed xml loads as an ErrorDescriptor.''' + + bad_xml = '''''' + + system = self.get_system() + + descriptor = XModuleDescriptor.load_from_xml(bad_xml, system, 'org', 'course', + None) + + self.assertEqual(descriptor.__class__.__name__, + 'ErrorDescriptor') + + def test_reimport(self): + '''Make sure an already-exported error xml tag loads properly''' + + self.maxDiff = None + bad_xml = '''''' + system = self.get_system() + descriptor = XModuleDescriptor.load_from_xml(bad_xml, system, 'org', 'course', + None) + resource_fs = None + tag_xml = descriptor.export_to_xml(resource_fs) + re_import_descriptor = XModuleDescriptor.load_from_xml(tag_xml, system, + 'org', 'course', + None) + self.assertEqual(re_import_descriptor.__class__.__name__, + 'ErrorDescriptor') + + self.assertEqual(descriptor.definition['data'], + re_import_descriptor.definition['data']) + + def test_fixed_xml_tag(self): + """Make sure a tag that's been fixed exports as the original tag type""" + + # create a error tag with valid xml contents + root = etree.Element('error') + good_xml = '''''' + root.text = good_xml + + xml_str_in = etree.tostring(root) + + # load it + system = self.get_system() + descriptor = XModuleDescriptor.load_from_xml(xml_str_in, system, 'org', 'course', + None) + # export it + resource_fs = None + xml_str_out = descriptor.export_to_xml(resource_fs) + + # Now make sure the exported xml is a sequential + xml_out = etree.fromstring(xml_str_out) + self.assertEqual(xml_out.tag, 'sequential') + + def test_metadata_inherit(self): + """Make sure metadata inherits properly""" + system = self.get_system() + v = "1 hour" + start_xml = ''' + + Two houses, ... + '''.format(grace=v) + descriptor = XModuleDescriptor.load_from_xml(start_xml, system, + 'org', 'course') + + print "Errors: {0}".format(system.errorlog.errors) + print descriptor, descriptor.metadata + self.assertEqual(descriptor.metadata['graceperiod'], v) + + # Check that the child inherits correctly + child = descriptor.get_children()[0] + self.assertEqual(child.metadata['graceperiod'], v) + + # Now export and see if the chapter tag has a graceperiod attribute + resource_fs = MemoryFS() + exported_xml = descriptor.export_to_xml(resource_fs) + print "Exported xml:", exported_xml + root = etree.fromstring(exported_xml) + chapter_tag = root[0] + self.assertEqual(chapter_tag.tag, 'chapter') + self.assertFalse('graceperiod' in chapter_tag.attrib) diff --git a/common/lib/xmodule/xmodule/tests/test_stringify.py b/common/lib/xmodule/xmodule/tests/test_stringify.py new file mode 100644 index 0000000000..1c6ee855f3 --- /dev/null +++ b/common/lib/xmodule/xmodule/tests/test_stringify.py @@ -0,0 +1,10 @@ +from nose.tools import assert_equals +from lxml import etree +from xmodule.stringify import stringify_children + +def test_stringify(): + text = 'Hi
      there Bruce!
      ' + html = '''{0}'''.format(text) + xml = etree.fromstring(html) + out = stringify_children(xml) + assert_equals(out, text) diff --git a/common/lib/xmodule/xmodule/video_module.py b/common/lib/xmodule/xmodule/video_module.py index da10e4bc91..fb68ba982b 100644 --- a/common/lib/xmodule/xmodule/video_module.py +++ b/common/lib/xmodule/xmodule/video_module.py @@ -23,11 +23,12 @@ class VideoModule(XModule): css = {'scss': [resource_string(__name__, 'css/video/display.scss')]} js_module_name = "Video" - def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs): - XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs) + def __init__(self, system, location, definition, + instance_state=None, shared_state=None, **kwargs): + XModule.__init__(self, system, location, definition, + instance_state, shared_state, **kwargs) xmltree = etree.fromstring(self.definition['data']) self.youtube = xmltree.get('youtube') - self.name = xmltree.get('name') self.position = 0 if instance_state is not None: @@ -71,7 +72,7 @@ class VideoModule(XModule): 'streams': self.video_list(), 'id': self.location.html_id(), 'position': self.position, - 'name': self.name, + 'display_name': self.display_name, # TODO (cpennington): This won't work when we move to data that isn't on the filesystem 'data_dir': self.metadata['data_dir'], }) diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index 0eaca8c426..60670767f7 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -1,10 +1,14 @@ -from lxml import etree -import pkg_resources import logging +import pkg_resources +import sys + +from fs.errors import ResourceNotFoundError +from functools import partial +from lxml import etree +from lxml.etree import XMLSyntaxError from xmodule.modulestore import Location - -from functools import partial +from xmodule.errortracker import exc_info_to_str log = logging.getLogger('mitx.' + __name__) @@ -187,13 +191,21 @@ class XModule(HTMLSnippet): self.instance_state = instance_state self.shared_state = shared_state self.id = self.location.url() - self.name = self.location.name + self.url_name = self.location.name self.category = self.location.category self.metadata = kwargs.get('metadata', {}) self._loaded_children = None - def get_name(self): - return self.name + @property + def display_name(self): + ''' + Return a display name for the module: use display_name if defined in + metadata, otherwise convert the url name. + ''' + return self.metadata.get('display_name', + self.url_name.replace('_', ' ')) + def __unicode__(self): + return '' % (self.name, self.category, self.id) def get_children(self): ''' @@ -338,6 +350,8 @@ class XModuleDescriptor(Plugin, HTMLSnippet): module display_name: The name to use for displaying this module to the user + url_name: The name to use for this module in urls and other places + where a unique name is needed. format: The format of this module ('Homework', 'Lab', etc) graded (bool): Whether this module is should be graded or not start (string): The date for which this module will be available @@ -352,13 +366,30 @@ class XModuleDescriptor(Plugin, HTMLSnippet): self.metadata = kwargs.get('metadata', {}) self.definition = definition if definition is not None else {} self.location = Location(kwargs.get('location')) - self.name = self.location.name + self.url_name = self.location.name self.category = self.location.category self.shared_state_key = kwargs.get('shared_state_key') self._child_instances = None self._inherited_metadata = set() + @property + def display_name(self): + ''' + Return a display name for the module: use display_name if defined in + metadata, otherwise convert the url name. + ''' + return self.metadata.get('display_name', + self.url_name.replace('_', ' ')) + + @property + def own_metadata(self): + """ + Return the metadata that is not inherited, but was defined on this module. + """ + return dict((k,v) for k,v in self.metadata.items() + if k not in self._inherited_metadata) + def inherit_metadata(self, metadata): """ Updates this module with metadata inherited from a containing module. @@ -443,16 +474,32 @@ class XModuleDescriptor(Plugin, HTMLSnippet): system is an XMLParsingSystem org and course are optional strings that will be used in the generated - modules url identifiers + module's url identifiers """ - class_ = XModuleDescriptor.load_class( - etree.fromstring(xml_data).tag, - default_class - ) - # leave next line, commented out - useful for low-level debugging - # log.debug('[XModuleDescriptor.load_from_xml] tag=%s, class_=%s' % ( - # etree.fromstring(xml_data).tag,class_)) - return class_.from_xml(xml_data, system, org, course) + try: + class_ = XModuleDescriptor.load_class( + etree.fromstring(xml_data).tag, + default_class + ) + # leave next line, commented out - useful for low-level debugging + # log.debug('[XModuleDescriptor.load_from_xml] tag=%s, class_=%s' % ( + # etree.fromstring(xml_data).tag,class_)) + + descriptor = class_.from_xml(xml_data, system, org, course) + except Exception as err: + # Didn't load properly. Fall back on loading as an error + # descriptor. This should never error due to formatting. + + # Put import here to avoid circular import errors + from xmodule.error_module import ErrorDescriptor + msg = "Error loading from xml." + log.exception(msg) + system.error_tracker(msg) + err_msg = msg + "\n" + exc_info_to_str(sys.exc_info()) + descriptor = ErrorDescriptor.from_xml(xml_data, system, org, course, + err_msg) + + return descriptor @classmethod def from_xml(cls, xml_data, system, org=None, course=None): @@ -521,16 +568,19 @@ class XModuleDescriptor(Plugin, HTMLSnippet): class DescriptorSystem(object): - def __init__(self, load_item, resources_fs, error_handler): + def __init__(self, load_item, resources_fs, error_tracker, **kwargs): """ load_item: Takes a Location and returns an XModuleDescriptor resources_fs: A Filesystem object that contains all of the resources needed for the course - error_handler: A hook for handling errors in loading the descriptor. - Must be a function of (error_msg, exc_info=None). - See errorhandlers.py for some simple ones. + error_tracker: A hook for tracking errors in loading the descriptor. + Used for example to get a list of all non-fatal problems on course + load, and display them to the user. + + A function of (error_msg). errortracker.py provides a + handy make_error_tracker() function. Patterns for using the error handler: try: @@ -539,10 +589,8 @@ class DescriptorSystem(object): except SomeProblem: msg = 'Grommet {0} is broken'.format(x) log.exception(msg) # don't rely on handler to log - self.system.error_handler(msg) - # if we get here, work around if possible - raise # if no way to work around - OR + self.system.error_tracker(msg) + # work around return 'Oops, couldn't load grommet' OR, if not in an exception context: @@ -550,25 +598,27 @@ class DescriptorSystem(object): if not check_something(thingy): msg = "thingy {0} is broken".format(thingy) log.critical(msg) - error_handler(msg) - # if we get here, work around - pass # e.g. if no workaround needed + self.system.error_tracker(msg) + + NOTE: To avoid duplication, do not call the tracker on errors + that you're about to re-raise---let the caller track them. """ self.load_item = load_item self.resources_fs = resources_fs - self.error_handler = error_handler + self.error_tracker = error_tracker class XMLParsingSystem(DescriptorSystem): - def __init__(self, load_item, resources_fs, error_handler, process_xml): + def __init__(self, load_item, resources_fs, error_tracker, process_xml, **kwargs): """ - load_item, resources_fs, error_handler: see DescriptorSystem + load_item, resources_fs, error_tracker: see DescriptorSystem process_xml: Takes an xml string, and returns a XModuleDescriptor created from that xml """ - DescriptorSystem.__init__(self, load_item, resources_fs, error_handler) + DescriptorSystem.__init__(self, load_item, resources_fs, error_tracker, + **kwargs) self.process_xml = process_xml @@ -584,10 +634,18 @@ class ModuleSystem(object): Note that these functions can be closures over e.g. a django request and user, or other environment-specific info. ''' - def __init__(self, ajax_url, track_function, - get_module, render_template, replace_urls, - user=None, filestore=None, debug=False, - xqueue=None): + def __init__(self, + ajax_url, + track_function, + get_module, + render_template, + replace_urls, + user=None, + filestore=None, + debug=False, + xqueue_callback_url=None, + xqueue_default_queuename="null", + is_staff=False): ''' Create a closure around the system environment. @@ -613,9 +671,13 @@ class ModuleSystem(object): replace_urls - TEMPORARY - A function like static_replace.replace_urls that capa_module can use to fix up the static urls in ajax results. + + is_staff - Is the user making the request a staff user? + TODO (vshnayder): this will need to change once we have real user roles. ''' self.ajax_url = ajax_url - self.xqueue = xqueue + self.xqueue_callback_url = xqueue_callback_url + self.xqueue_default_queuename = xqueue_default_queuename self.track_function = track_function self.filestore = filestore self.get_module = get_module @@ -623,6 +685,7 @@ class ModuleSystem(object): self.DEBUG = self.debug = debug self.seed = user.id if user is not None else 0 self.replace_urls = replace_urls + self.is_staff = is_staff def get(self, attr): ''' provide uniform access to attributes (like etree).''' diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index 6750906eb4..7a12ed869d 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -1,4 +1,3 @@ -from collections import MutableMapping from xmodule.x_module import XModuleDescriptor from xmodule.modulestore import Location from lxml import etree @@ -8,74 +7,12 @@ import traceback from collections import namedtuple from fs.errors import ResourceNotFoundError import os +import sys log = logging.getLogger(__name__) -# TODO (cpennington): This was implemented in an attempt to improve performance, -# but the actual improvement wasn't measured (and it was implemented late at night). -# We should check if it hurts, and whether there's a better way of doing lazy loading - - -class LazyLoadingDict(MutableMapping): - """ - A dictionary object that lazily loads its contents from a provided - function on reads (of members that haven't already been set). - """ - - def __init__(self, loader): - ''' - On the first read from this dictionary, it will call loader() to - populate its contents. loader() must return something dict-like. Any - elements set before the first read will be preserved. - ''' - self._contents = {} - self._loaded = False - self._loader = loader - self._deleted = set() - - def __getitem__(self, name): - if not (self._loaded or name in self._contents or name in self._deleted): - self.load() - - return self._contents[name] - - def __setitem__(self, name, value): - self._contents[name] = value - self._deleted.discard(name) - - def __delitem__(self, name): - del self._contents[name] - self._deleted.add(name) - - def __contains__(self, name): - self.load() - return name in self._contents - - def __len__(self): - self.load() - return len(self._contents) - - def __iter__(self): - self.load() - return iter(self._contents) - - def __repr__(self): - self.load() - return repr(self._contents) - - def load(self): - if self._loaded: - return - - loaded_contents = self._loader() - loaded_contents.update(self._contents) - self._contents = loaded_contents - self._loaded = True - - _AttrMapBase = namedtuple('_AttrMap', 'metadata_key to_metadata from_metadata') - class AttrMap(_AttrMapBase): """ A class that specifies a metadata_key, and two functions: @@ -104,6 +41,7 @@ class XmlDescriptor(XModuleDescriptor): # to definition_from_xml, and from the xml returned by definition_to_xml metadata_attributes = ('format', 'graceperiod', 'showanswer', 'rerandomize', 'start', 'due', 'graded', 'display_name', 'url_name', 'hide_from_toc', + 'ispublic', # if True, then course is listed for all users; see # VS[compat] Remove once unused. 'name', 'slug') @@ -163,6 +101,52 @@ class XmlDescriptor(XModuleDescriptor): """ return etree.parse(file_object).getroot() + @classmethod + def load_definition(cls, xml_object, system, location): + '''Load a descriptor definition from the specified xml_object. + Subclasses should not need to override this except in special + cases (e.g. html module)''' + + filename = xml_object.get('filename') + if filename is None: + definition_xml = copy.deepcopy(xml_object) + filepath = '' + else: + filepath = cls._format_filepath(xml_object.tag, filename) + + # VS[compat] + # TODO (cpennington): If the file doesn't exist at the right path, + # give the class a chance to fix it up. The file will be written out + # again in the correct format. This should go away once the CMS is + # online and has imported all current (fall 2012) courses from xml + if not system.resources_fs.exists(filepath) and hasattr( + cls, + 'backcompat_paths'): + candidates = cls.backcompat_paths(filepath) + for candidate in candidates: + if system.resources_fs.exists(candidate): + filepath = candidate + break + + try: + with system.resources_fs.open(filepath) as file: + definition_xml = cls.file_to_xml(file) + except Exception: + msg = 'Unable to load file contents at path %s for item %s' % ( + filepath, location.url()) + # Add info about where we are, but keep the traceback + raise Exception, msg, sys.exc_info()[2] + + cls.clean_metadata_from_xml(definition_xml) + definition = cls.definition_from_xml(definition_xml, system) + + # TODO (ichuang): remove this after migration + # for Fall 2012 LMS migration: keep filename (and unmangled filename) + definition['filename'] = [ filepath, filename ] + + return definition + + @classmethod def from_xml(cls, xml_data, system, org=None, course=None): """ @@ -180,7 +164,7 @@ class XmlDescriptor(XModuleDescriptor): slug = xml_object.get('url_name', xml_object.get('slug')) location = Location('i4x', org, course, xml_object.tag, slug) - def metadata_loader(): + def load_metadata(): metadata = {} for attr in cls.metadata_attributes: val = xml_object.get(attr) @@ -192,49 +176,15 @@ class XmlDescriptor(XModuleDescriptor): metadata[attr_map.metadata_key] = attr_map.to_metadata(val) return metadata - def definition_loader(): - filename = xml_object.get('filename') - if filename is None: - definition_xml = copy.deepcopy(xml_object) - else: - filepath = cls._format_filepath(xml_object.tag, filename) - - # VS[compat] - # TODO (cpennington): If the file doesn't exist at the right path, - # give the class a chance to fix it up. The file will be written out again - # in the correct format. - # This should go away once the CMS is online and has imported all current (fall 2012) - # courses from xml - if not system.resources_fs.exists(filepath) and hasattr(cls, 'backcompat_paths'): - candidates = cls.backcompat_paths(filepath) - for candidate in candidates: - if system.resources_fs.exists(candidate): - filepath = candidate - break - - try: - with system.resources_fs.open(filepath) as file: - definition_xml = cls.file_to_xml(file) - except (ResourceNotFoundError, etree.XMLSyntaxError): - msg = 'Unable to load file contents at path %s for item %s' % (filepath, location.url()) - log.exception(msg) - system.error_handler(msg) - # if error_handler didn't reraise, work around problem. - error_elem = etree.Element('error') - message_elem = etree.SubElement(error_elem, 'error_message') - message_elem.text = msg - stack_elem = etree.SubElement(error_elem, 'stack_trace') - stack_elem.text = traceback.format_exc() - return {'data': etree.tostring(error_elem)} - - cls.clean_metadata_from_xml(definition_xml) - return cls.definition_from_xml(definition_xml, system) - + definition = cls.load_definition(xml_object, system, location) + metadata = load_metadata() + # VS[compat] -- just have the url_name lookup once translation is done + slug = xml_object.get('url_name', xml_object.get('slug')) return cls( system, - LazyLoadingDict(definition_loader), + definition, location=location, - metadata=LazyLoadingDict(metadata_loader), + metadata=metadata, ) @classmethod @@ -282,8 +232,8 @@ class XmlDescriptor(XModuleDescriptor): # Write it to a file if necessary if self.split_to_file(xml_object): - # Put this object in it's own file - filepath = self.__class__._format_filepath(self.category, self.name) + # Put this object in its own file + filepath = self.__class__._format_filepath(self.category, self.url_name) resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True) with resource_fs.open(filepath, 'w') as file: file.write(etree.tostring(xml_object, pretty_print=True)) @@ -296,10 +246,10 @@ class XmlDescriptor(XModuleDescriptor): xml_object.tail = '' - xml_object.set('filename', self.name) + xml_object.set('filename', self.url_name) # Add the metadata - xml_object.set('url_name', self.name) + xml_object.set('url_name', self.url_name) for attr in self.metadata_attributes: attr_map = self.xml_attribute_map.get(attr, AttrMap(attr)) metadata_key = attr_map.metadata_key diff --git a/create-dev-env.sh b/create-dev-env.sh index 91f9b81931..8a1a7e6f89 100755 --- a/create-dev-env.sh +++ b/create-dev-env.sh @@ -17,8 +17,10 @@ ouch() { !! ERROR !! The last command did not complete successfully, - see $LOG for more details or trying running the + For more details or trying running the script again with the -v flag. + + Output of the script is recorded in $LOG EOL printf '\E[0m' @@ -36,7 +38,7 @@ usage() { Usage: $PROG [-c] [-v] [-h] -c compile scipy and numpy - -s do _not_ set --no-site-packages for virtualenv + -s give access to global site-packages for virtualenv -v set -x + spew -h this @@ -61,28 +63,21 @@ clone_repos() { if [[ -d "$BASE/mitx/.git" ]]; then output "Pulling mitx" cd "$BASE/mitx" - git pull >>$LOG + git pull else output "Cloning mitx" if [[ -d "$BASE/mitx" ]]; then mv "$BASE/mitx" "${BASE}/mitx.bak.$$" fi - git clone git@github.com:MITx/mitx.git >>$LOG + git clone git@github.com:MITx/mitx.git + fi + + if [[ ! -d "$BASE/mitx/askbot/.git" ]]; then + output "Cloning askbot as a submodule of mitx" + cd "$BASE/mitx" + git submodule update --init fi - cd "$BASE" - if [[ -d "$BASE/askbot-devel/.git" ]]; then - output "Pulling askbot-devel" - cd "$BASE/askbot-devel" - git pull >>$LOG - else - output "Cloning askbot-devel" - if [[ -d "$BASE/askbot-devel" ]]; then - mv "$BASE/askbot-devel" "${BASE}/askbot-devel.bak.$$" - fi - git clone git@github.com:MITx/askbot-devel >>$LOG - fi - # By default, dev environments start with a copy of 6.002x cd "$BASE" mkdir -p "$BASE/data" @@ -90,14 +85,14 @@ clone_repos() { if [[ -d "$BASE/data/$REPO/.git" ]]; then output "Pulling $REPO" cd "$BASE/data/$REPO" - git pull >>$LOG + git pull else output "Cloning $REPO" if [[ -d "$BASE/data/$REPO" ]]; then mv "$BASE/data/$REPO" "${BASE}/data/$REPO.bak.$$" fi cd "$BASE/data" - git clone git@github.com:MITx/$REPO >>$LOG + git clone git@github.com:MITx/$REPO fi } @@ -109,8 +104,8 @@ RUBY_VER="1.9.3" NUMPY_VER="1.6.2" SCIPY_VER="0.10.1" BREW_FILE="$BASE/mitx/brew-formulas.txt" -LOG="/var/tmp/install.log" -APT_PKGS="curl git mercurial python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript" +LOG="/var/tmp/install-$(date +%Y%m%d-%H%M%S).log" +APT_PKGS="curl git python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript" if [[ $EUID -eq 0 ]]; then error "This script should not be run using sudo or as the root user" @@ -163,23 +158,30 @@ cat< >(tee $LOG) +exec 2>&1 + +if ! grep -q "export rvm_path=$RUBY_DIR" ~/.rvmrc; then + if [[ -f $HOME/.rvmrc ]]; then + output "Copying existing .rvmrc to .rvmrc.bak" + cp $HOME/.rvmrc $HOME/.rvmrc.bak + fi + output "Creating $HOME/.rvmrc so rvm uses $RUBY_DIR" echo "export rvm_path=$RUBY_DIR" > $HOME/.rvmrc fi + mkdir -p $BASE -rm -f $LOG case `uname -s` in [Ll]inux) command -v lsb_release &>/dev/null || { @@ -201,17 +203,31 @@ case `uname -s` in esac ;; Darwin) + + if [[ ! -w /usr/local ]]; then + cat</dev/null || { output "Installing brew" /usr/bin/ruby -e "$(curl -fsSL https://raw.github.com/mxcl/homebrew/master/Library/Contributions/install_homebrew.rb)" } command -v git &>/dev/null || { output "Installing git" - brew install git >> $LOG - } - command -v hg &>/dev/null || { - output "Installaing mercurial" - brew install mercurial >> $LOG + brew install git } clone_repos @@ -225,20 +241,22 @@ case `uname -s` in for pkg in $(cat $BREW_FILE); do grep $pkg <(brew list) &>/dev/null || { output "Installing $pkg" - brew install $pkg >>$LOG + brew install $pkg } done command -v pip &>/dev/null || { output "Installing pip" - sudo easy_install pip >>$LOG - } - command -v virtualenv &>/dev/null || { - output "Installing virtualenv" - sudo pip install virtualenv virtualenvwrapper >> $LOG + sudo easy_install pip } + + if ! grep -Eq ^1.7 <(virtualenv --version 2>/dev/null); then + output "Installing virtualenv >1.7" + sudo pip install 'virtualenv>1.7' virtualenvwrapper + fi + command -v coffee &>/dev/null || { output "Installing coffee script" - curl http://npmjs.org/install.sh | sh + curl --insecure https://npmjs.org/install.sh | sh npm install -g coffee-script } ;; @@ -253,10 +271,12 @@ curl -sL get.rvm.io | bash -s stable source $RUBY_DIR/scripts/rvm # skip the intro LESS="-E" rvm install $RUBY_VER -if [[ -n $systempkgs ]]; then - virtualenv "$PYTHON_DIR" +if [[ $systempkgs ]]; then + virtualenv --system-site-packages "$PYTHON_DIR" else - virtualenv --no-site-packages "$PYTHON_DIR" + # default behavior for virtualenv>1.7 is + # --no-site-packages + virtualenv "$PYTHON_DIR" fi source $PYTHON_DIR/bin/activate output "Installing gem bundler" @@ -277,24 +297,24 @@ if [[ -n $compile ]]; then rm -f numpy.tar.gz scipy.tar.gz output "Compiling numpy" cd "$BASE/numpy-${NUMPY_VER}" - python setup.py install >>$LOG 2>&1 + python setup.py install output "Compiling scipy" cd "$BASE/scipy-${SCIPY_VER}" - python setup.py install >>$LOG 2>&1 + python setup.py install cd "$BASE" rm -rf numpy-${NUMPY_VER} scipy-${SCIPY_VER} fi -output "Installing askbot requirements" -pip install -r askbot-devel/askbot_requirements.txt >>$LOG -output "Installing askbot-dev requirements" -pip install -r askbot-devel/askbot_requirements_dev.txt >>$LOG output "Installing MITx pre-requirements" -pip install -r mitx/pre-requirements.txt >> $LOG +pip install -r mitx/pre-requirements.txt # Need to be in the mitx dir to get the paths to local modules right output "Installing MITx requirements" cd mitx -pip install -r requirements.txt >>$LOG +pip install -r requirements.txt +output "Installing askbot requirements" +pip install -r askbot/askbot_requirements.txt +pip install -r askbot/askbot_requirements_dev.txt + mkdir "$BASE/log" || true mkdir "$BASE/db" || true diff --git a/lms/askbot/skins/mitx/templates/base.html b/lms/askbot/skins/mitx/templates/base.html index ced2376a99..18ca213cb7 100644 --- a/lms/askbot/skins/mitx/templates/base.html +++ b/lms/askbot/skins/mitx/templates/base.html @@ -20,7 +20,7 @@ {% include "widgets/header.html" %} {# Logo, user tool navigation and meta navitation #} {# include "widgets/secondary_header.html" #} {# Scope selector, search input and ask button #} -
      +
      {% block body %} {% endblock %}
      diff --git a/lms/askbot/skins/mitx/templates/meta/html_head_stylesheets.html b/lms/askbot/skins/mitx/templates/meta/html_head_stylesheets.html index 99223edac4..3ec11b59fd 100644 --- a/lms/askbot/skins/mitx/templates/meta/html_head_stylesheets.html +++ b/lms/askbot/skins/mitx/templates/meta/html_head_stylesheets.html @@ -1,3 +1,4 @@ {% load extra_filters_jinja %} {{ 'application' | compressed_css }} +{{ 'course' | compressed_css }} diff --git a/lms/askbot/skins/mitx/templates/navigation.jinja.html b/lms/askbot/skins/mitx/templates/navigation.jinja.html index d69afbebc6..59c7148184 100644 --- a/lms/askbot/skins/mitx/templates/navigation.jinja.html +++ b/lms/askbot/skins/mitx/templates/navigation.jinja.html @@ -1,46 +1,27 @@ -
      +
      diff --git a/lms/askbot/skins/mitx/templates/widgets/footer.html b/lms/askbot/skins/mitx/templates/widgets/footer.html index 4675a07047..d152e2a38e 100644 --- a/lms/askbot/skins/mitx/templates/widgets/footer.html +++ b/lms/askbot/skins/mitx/templates/widgets/footer.html @@ -1,26 +1,38 @@ - + +
      + +
      +
      +

      © 2012 edX, some rights reserved.

      +
      + +
      + Terms of Service + Privacy Policy + Honor Code + Help +
      +
      + + + diff --git a/lms/djangoapps/courseware/course_settings.py b/lms/djangoapps/courseware/course_settings.py deleted file mode 100644 index 5b2348bee6..0000000000 --- a/lms/djangoapps/courseware/course_settings.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -Course settings module. All settings in the global_settings are -first applied, and then any settings in the settings.DATA_DIR/course_settings.json -are applied. A setting must be in ALL_CAPS. - -Settings are used by calling - -from courseware.course_settings import course_settings - -Note that courseware.course_settings.course_settings is not a module -- it's an object. So -importing individual settings is not possible: - -from courseware.course_settings.course_settings import GRADER # This won't work. - -""" -import json -import logging - -from django.conf import settings - -from xmodule import graders - -log = logging.getLogger("mitx.courseware") - -global_settings_json = """ -{ - "GRADER" : [ - { - "type" : "Homework", - "min_count" : 12, - "drop_count" : 2, - "short_label" : "HW", - "weight" : 0.15 - }, - { - "type" : "Lab", - "min_count" : 12, - "drop_count" : 2, - "category" : "Labs", - "weight" : 0.15 - }, - { - "type" : "Midterm", - "name" : "Midterm Exam", - "short_label" : "Midterm", - "weight" : 0.3 - }, - { - "type" : "Final", - "name" : "Final Exam", - "short_label" : "Final", - "weight" : 0.4 - } - ], - "GRADE_CUTOFFS" : { - "A" : 0.87, - "B" : 0.7, - "C" : 0.6 - } -} -""" - - -class Settings(object): - def __init__(self): - - # Load the global settings as a dictionary - global_settings = json.loads(global_settings_json) - - # Load the course settings as a dictionary - course_settings = {} - try: - # TODO: this doesn't work with multicourse - with open(settings.DATA_DIR + "/course_settings.json") as course_settings_file: - course_settings_string = course_settings_file.read() - course_settings = json.loads(course_settings_string) - except IOError: - log.warning("Unable to load course settings file from " + str(settings.DATA_DIR) + "/course_settings.json") - - # Override any global settings with the course settings - global_settings.update(course_settings) - - # Now, set the properties from the course settings on ourselves - for setting in global_settings: - setting_value = global_settings[setting] - setattr(self, setting, setting_value) - - # Here is where we should parse any configurations, so that we can fail early - self.GRADER = graders.grader_from_conf(self.GRADER) - -course_settings = Settings() diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index c2c391b08e..8193988d67 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -1,3 +1,4 @@ +from collections import defaultdict from fs.errors import ResourceNotFoundError from functools import wraps import logging @@ -33,6 +34,7 @@ def check_course(course_id, course_must_be_open=True, course_required=True): try: course_loc = CourseDescriptor.id_to_location(course_id) course = modulestore().get_item(course_loc) + except (KeyError, ItemNotFoundError): raise Http404("Course not found.") @@ -82,7 +84,7 @@ def get_course_about_section(course, section_key): log.warning("Missing about section {key} in course {url}".format(key=section_key, url=course.location.url())) return None elif section_key == "title": - return course.metadata.get('display_name', course.name) + return course.metadata.get('display_name', course.url_name) elif section_key == "university": return course.location.org elif section_key == "number": @@ -113,3 +115,57 @@ def get_course_info_section(course, section_key): return "! Info section missing !" raise KeyError("Invalid about key " + str(section_key)) + +def course_staff_group_name(course): + ''' + course should be either a CourseDescriptor instance, or a string (the .course entry of a Location) + ''' + if isinstance(course,str): + coursename = course + else: + coursename = course.metadata.get('data_dir','UnknownCourseName') + if not coursename: # Fall 2012: not all course.xml have metadata correct yet + coursename = course.metadata.get('course','') + return 'staff_%s' % coursename + +def has_staff_access_to_course(user,course): + ''' + Returns True if the given user has staff access to the course. + This means that user is in the staff_* group, or is an overall admin. + ''' + if user is None or (not user.is_authenticated()) or course is None: + return False + if user.is_staff: + return True + user_groups = [x[1] for x in user.groups.values_list()] # note this is the Auth group, not UserTestGroup + staff_group = course_staff_group_name(course) + log.debug('course %s user %s groups %s' % (staff_group, user, user_groups)) + if staff_group in user_groups: + return True + return False + +def has_access_to_course(user,course): + if course.metadata.get('ispublic'): + return True + return has_staff_access_to_course(user,course) + +def get_courses_by_university(user): + ''' + Returns dict of lists of courses available, keyed by course.org (ie university). + Courses are sorted by course.number. + + if ACCESS_REQUIRE_STAFF_FOR_COURSE then list only includes those accessible to user. + ''' + # TODO: Clean up how 'error' is done. + # filter out any courses that errored. + courses = [c for c in modulestore().get_courses() + if isinstance(c, CourseDescriptor)] + courses = sorted(courses, key=lambda course: course.number) + universities = defaultdict(list) + for course in courses: + if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'): + if not has_access_to_course(user,course): + continue + universities[course.org].append(course) + return universities + diff --git a/lms/djangoapps/courseware/grades.py b/lms/djangoapps/courseware/grades.py index 66aff08dca..6e7a0ce102 100644 --- a/lms/djangoapps/courseware/grades.py +++ b/lms/djangoapps/courseware/grades.py @@ -3,7 +3,6 @@ import logging from django.conf import settings -from courseware.course_settings import course_settings from xmodule import graders from xmodule.graders import Score from models import StudentModule @@ -11,20 +10,25 @@ from models import StudentModule _log = logging.getLogger("mitx.courseware") -def grade_sheet(student, course, student_module_cache): +def grade_sheet(student, course, grader, student_module_cache): """ - This pulls a summary of all problems in the course. It returns a dictionary with two datastructures: + This pulls a summary of all problems in the course. It returns a dictionary + with two datastructures: - - courseware_summary is a summary of all sections with problems in the course. It is organized as an array of chapters, - each containing an array of sections, each containing an array of scores. This contains information for graded and ungraded - problems, and is good for displaying a course summary with due dates, etc. + - courseware_summary is a summary of all sections with problems in the + course. It is organized as an array of chapters, each containing an array of + sections, each containing an array of scores. This contains information for + graded and ungraded problems, and is good for displaying a course summary + with due dates, etc. - - grade_summary is the output from the course grader. More information on the format is in the docstring for CourseGrader. + - grade_summary is the output from the course grader. More information on + the format is in the docstring for CourseGrader. Arguments: student: A User object for the student to grade course: An XModule containing the course to grade - student_module_cache: A StudentModuleCache initialized with all instance_modules for the student + student_module_cache: A StudentModuleCache initialized with all + instance_modules for the student """ totaled_scores = {} chapters = [] @@ -52,12 +56,16 @@ def grade_sheet(student, course, student_module_cache): correct = total if not total > 0: - #We simply cannot grade a problem that is 12/0, because we might need it as a percentage + #We simply cannot grade a problem that is 12/0, because we + #might need it as a percentage graded = False - scores.append(Score(correct, total, graded, module.metadata.get('display_name'))) + scores.append(Score(correct, total, graded, + module.metadata.get('display_name'))) + + section_total, graded_total = graders.aggregate_scores( + scores, s.metadata.get('display_name')) - section_total, graded_total = graders.aggregate_scores(scores, s.metadata.get('display_name')) #Add the graded total to totaled_scores format = s.metadata.get('format', "") if format and graded_total.possible > 0: @@ -66,7 +74,8 @@ def grade_sheet(student, course, student_module_cache): totaled_scores[format] = format_scores sections.append({ - 'section': s.metadata.get('display_name'), + 'display_name': s.display_name, + 'url_name': s.url_name, 'scores': scores, 'section_total': section_total, 'format': format, @@ -74,11 +83,11 @@ def grade_sheet(student, course, student_module_cache): 'graded': graded, }) - chapters.append({'course': course.metadata.get('display_name'), - 'chapter': c.metadata.get('display_name'), + chapters.append({'course': course.display_name, + 'display_name': c.display_name, + 'url_name': c.url_name, 'sections': sections}) - grader = course_settings.GRADER grade_summary = grader.grade(totaled_scores) return {'courseware_summary': chapters, diff --git a/lms/djangoapps/courseware/management/commands/clean_xml.py b/lms/djangoapps/courseware/management/commands/clean_xml.py index 7523fd8373..de845df572 100644 --- a/lms/djangoapps/courseware/management/commands/clean_xml.py +++ b/lms/djangoapps/courseware/management/commands/clean_xml.py @@ -10,37 +10,17 @@ from lxml import etree from django.core.management.base import BaseCommand from xmodule.modulestore.xml import XMLModuleStore - +from xmodule.errortracker import make_error_tracker def traverse_tree(course): '''Load every descriptor in course. Return bool success value.''' queue = [course] while len(queue) > 0: node = queue.pop() -# print '{0}:'.format(node.location) -# if 'data' in node.definition: -# print '{0}'.format(node.definition['data']) queue.extend(node.get_children()) return True -def make_logging_error_handler(): - '''Return a tuple (handler, error_list), where - the handler appends the message and any exc_info - to the error_list on every call. - ''' - errors = [] - - def error_handler(msg, exc_info=None): - '''Log errors''' - if exc_info is None: - if sys.exc_info() != (None, None, None): - exc_info = sys.exc_info() - - errors.append((msg, exc_info)) - - return (error_handler, errors) - def export(course, export_dir): """Export the specified course to course_dir. Creates dir if it doesn't exist. @@ -73,32 +53,18 @@ def import_with_checks(course_dir, verbose=True): data_dir = course_dir.dirname() course_dirs = [course_dir.basename()] - (error_handler, errors) = make_logging_error_handler() # No default class--want to complain if it doesn't find plugins for any # module. modulestore = XMLModuleStore(data_dir, default_class=None, eager=True, - course_dirs=course_dirs, - error_handler=error_handler) + course_dirs=course_dirs) def str_of_err(tpl): - (msg, exc_info) = tpl - if exc_info is None: - return msg - - exc_str = '\n'.join(traceback.format_exception(*exc_info)) + (msg, exc_str) = tpl return '{msg}\n{exc}'.format(msg=msg, exc=exc_str) courses = modulestore.get_courses() - if len(errors) != 0: - all_ok = False - print '\n' - print "=" * 40 - print 'ERRORs during import:' - print '\n'.join(map(str_of_err,errors)) - print "=" * 40 - print '\n' n = len(courses) if n != 1: @@ -107,6 +73,16 @@ def import_with_checks(course_dir, verbose=True): return (False, None) course = courses[0] + errors = modulestore.get_item_errors(course.location) + if len(errors) != 0: + all_ok = False + print '\n' + print "=" * 40 + print 'ERRORs during import:' + print '\n'.join(map(str_of_err, errors)) + print "=" * 40 + print '\n' + #print course validators = ( @@ -143,6 +119,7 @@ def check_roundtrip(course_dir): # dircmp doesn't do recursive diffs. # diff = dircmp(course_dir, export_dir, ignore=[], hide=[]) print "======== Roundtrip diff: =========" + sys.stdout.flush() # needed to make diff appear in the right place os.system("diff -r {0} {1}".format(course_dir, export_dir)) print "======== ideally there is no diff above this =======" diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index b54e71df6d..5b4dc602e2 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -16,6 +16,8 @@ from xmodule.exceptions import NotFoundError from xmodule.x_module import ModuleSystem from xmodule_modifiers import replace_static_urls, add_histogram, wrap_xmodule +from courseware.courses import has_staff_access_to_course + log = logging.getLogger("mitx.courseware") @@ -36,10 +38,12 @@ def toc_for_course(user, request, course, active_chapter, active_section): Create a table of contents from the module store Return format: - [ {'name': name, 'sections': SECTIONS, 'active': bool}, ... ] + [ {'display_name': name, 'url_name': url_name, + 'sections': SECTIONS, 'active': bool}, ... ] where SECTIONS is a list - [ {'name': name, 'format': format, 'due': due, 'active' : bool}, ...] + [ {'display_name': name, 'url_name': url_name, + 'format': format, 'due': due, 'active' : bool}, ...] active is set for the section and chapter corresponding to the passed parameters. Everything else comes from the xml, or defaults to "". @@ -55,19 +59,21 @@ def toc_for_course(user, request, course, active_chapter, active_section): sections = list() for section in chapter.get_display_items(): - active = (chapter.metadata.get('display_name') == active_chapter and - section.metadata.get('display_name') == active_section) + active = (chapter.display_name == active_chapter and + section.display_name == active_section) hide_from_toc = section.metadata.get('hide_from_toc', 'false').lower() == 'true' if not hide_from_toc: - sections.append({'name': section.metadata.get('display_name'), + sections.append({'display_name': section.display_name, + 'url_name': section.url_name, 'format': section.metadata.get('format', ''), 'due': section.metadata.get('due', ''), 'active': active}) - chapters.append({'name': chapter.metadata.get('display_name'), + chapters.append({'display_name': chapter.display_name, + 'url_name': chapter.url_name, 'sections': sections, - 'active': chapter.metadata.get('display_name') == active_chapter}) + 'active': chapter.display_name == active_chapter}) return chapters @@ -77,8 +83,8 @@ def get_section(course_module, chapter, section): or None if this doesn't specify a valid section course: Course url - chapter: Chapter name - section: Section name + chapter: Chapter url_name + section: Section url_name """ if course_module is None: @@ -86,7 +92,7 @@ def get_section(course_module, chapter, section): chapter_module = None for _chapter in course_module.get_children(): - if _chapter.metadata.get('display_name') == chapter: + if _chapter.url_name == chapter: chapter_module = _chapter break @@ -95,7 +101,7 @@ def get_section(course_module, chapter, section): section_module = None for _section in chapter_module.get_children(): - if _section.metadata.get('display_name') == section: + if _section.url_name == section: section_module = _section break @@ -142,12 +148,12 @@ def get_module(user, request, location, student_module_cache, position=None): # Setup system context for module instance ajax_url = settings.MITX_ROOT_URL + '/modx/' + descriptor.location.url() + '/' - # Fully qualified callback URL for external queueing system - xqueue_callback_url = (request.build_absolute_uri('/') + settings.MITX_ROOT_URL + - 'xqueue/' + str(user.id) + '/' + descriptor.location.url() + '/' + + # Fully qualified callback URL for external queueing system + xqueue_callback_url = (request.build_absolute_uri('/') + settings.MITX_ROOT_URL + + 'xqueue/' + str(user.id) + '/' + descriptor.location.url() + '/' + 'score_update') - # Default queuename is course-specific and is derived from the course that + # Default queuename is course-specific and is derived from the course that # contains the current module. # TODO: Queuename should be derived from 'course_settings.json' of each course xqueue_default_queuename = descriptor.location.org + '-' + descriptor.location.course @@ -176,6 +182,7 @@ def get_module(user, request, location, student_module_cache, position=None): # a module is coming through get_html and is therefore covered # by the replace_static_urls code below replace_urls=replace_urls, + is_staff=user.is_staff, ) # pass position specified in URL to module through ModuleSystem system.set('position', position) @@ -187,8 +194,9 @@ def get_module(user, request, location, student_module_cache, position=None): module.metadata['data_dir'] ) - if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF') and user.is_staff: - module.get_html = add_histogram(module.get_html) + if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF'): + if has_staff_access_to_course(user, module.location.course): + module.get_html = add_histogram(module.get_html, module) # If StudentModule for this instance wasn't already in the database, # and this isn't a guest user, create it. diff --git a/lms/djangoapps/courseware/tests/tests.py b/lms/djangoapps/courseware/tests/tests.py index 8e9d13f8d5..0e2f7d7aa5 100644 --- a/lms/djangoapps/courseware/tests/tests.py +++ b/lms/djangoapps/courseware/tests/tests.py @@ -1,19 +1,20 @@ import copy import json +from path import path import os from pprint import pprint +from nose import SkipTest from django.test import TestCase from django.test.client import Client -from mock import patch, Mock -from override_settings import override_settings from django.conf import settings from django.core.urlresolvers import reverse -from path import path +from mock import patch, Mock +from override_settings import override_settings -from student.models import Registration from django.contrib.auth.models import User +from student.models import Registration from xmodule.modulestore.django import modulestore import xmodule.modulestore.django @@ -189,11 +190,12 @@ class RealCoursesLoadTestCase(PageLoader): xmodule.modulestore.django._MODULESTORES = {} xmodule.modulestore.django.modulestore().collection.drop() - # TODO: Disabled test for now.. Fix once things are cleaned up. - def Xtest_real_courses_loads(self): + def test_real_courses_loads(self): '''See if any real courses are available at the REAL_DATA_DIR. If they are, check them.''' + # TODO: Disabled test for now.. Fix once things are cleaned up. + raise SkipTest # TODO: adjust staticfiles_dirs if not os.path.isdir(REAL_DATA_DIR): # No data present. Just pass. diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py index 00fde8a84c..f014e3fcb5 100644 --- a/lms/djangoapps/courseware/views.py +++ b/lms/djangoapps/courseware/views.py @@ -1,4 +1,3 @@ -from collections import defaultdict import json import logging import urllib @@ -19,8 +18,8 @@ from django.views.decorators.cache import cache_control from module_render import toc_for_course, get_module, get_section from models import StudentModuleCache from student.models import UserProfile -from multicourse import multicourse_settings from xmodule.modulestore import Location +from xmodule.modulestore.search import path_to_location from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, NoPathToItem from xmodule.modulestore.django import modulestore from xmodule.course_module import CourseDescriptor @@ -28,7 +27,7 @@ from xmodule.course_module import CourseDescriptor from util.cache import cache, cache_if_anonymous from student.models import UserTestGroup, CourseEnrollment from courseware import grades -from courseware.courses import check_course +from courseware.courses import check_course, get_courses_by_university log = logging.getLogger("mitx.courseware") @@ -54,22 +53,16 @@ def user_groups(user): return group_names -def format_url_params(params): - return [urllib.quote(string.replace(' ', '_')) for string in params] - @ensure_csrf_cookie @cache_if_anonymous def courses(request): - # TODO: Clean up how 'error' is done. - courses = sorted(modulestore().get_courses(), key=lambda course: course.number) - universities = defaultdict(list) - for course in courses: - universities[course.org].append(course) - + ''' + Render "find courses" page. The course selection work is done in courseware.courses. + ''' + universities = get_courses_by_university(request.user) return render_to_response("courses.html", {'universities': universities}) - @cache_control(no_cache=True, no_store=True, must_revalidate=True) def gradebook(request, course_id): if 'course_admin' not in user_groups(request.user): @@ -110,7 +103,7 @@ def profile(request, course_id, student_id=None): user_info = UserProfile.objects.get(user=student) student_module_cache = StudentModuleCache(request.user, course) - course, _, _, _ = get_module(request.user, request, course.location, student_module_cache) + course_module, _, _, _ = get_module(request.user, request, course.location, student_module_cache) context = {'name': user_info.name, 'username': student.username, @@ -118,10 +111,9 @@ def profile(request, course_id, student_id=None): 'language': user_info.language, 'email': student.email, 'course': course, - 'format_url_params': format_url_params, 'csrf': csrf(request)['csrf_token'] } - context.update(grades.grade_sheet(student, course, student_module_cache)) + context.update(grades.grade_sheet(student, course_module, course.grader, student_module_cache)) return render_to_response('profile.html', context) @@ -132,9 +124,9 @@ def render_accordion(request, course, chapter, section): If chapter and section are '' or None, renders a default accordion. - Returns (initialization_javascript, content)''' + Returns the html string''' - # TODO (cpennington): do the right thing with courses + # grab the table of contents toc = toc_for_course(request.user, request, course, chapter, section) active_chapter = 1 @@ -146,11 +138,11 @@ def render_accordion(request, course, chapter, section): ('toc', toc), ('course_name', course.title), ('course_id', course.id), - ('format_url_params', format_url_params), ('csrf', csrf(request)['csrf_token'])] + template_imports.items()) return render_to_string('accordion.html', context) +@login_required @ensure_csrf_cookie @cache_control(no_cache=True, no_store=True, must_revalidate=True) def index(request, course_id, chapter=None, section=None, @@ -163,9 +155,9 @@ def index(request, course_id, chapter=None, section=None, Arguments: - request : HTTP request - - course : coursename (str) - - chapter : chapter name (str) - - section : section name (str) + - course_id : course id (str: ORG/course/URL_NAME) + - chapter : chapter url_name (str) + - section : section url_name (str) - position : position in module, eg of module (str) Returns: @@ -173,50 +165,63 @@ def index(request, course_id, chapter=None, section=None, - HTTPresponse ''' course = check_course(course_id) + registered = registered_for_course(course, request.user) + if not registered: + log.debug('User %s tried to view course %s but is not enrolled' % (request.user,course.location.url())) + return redirect(reverse('about_course', args=[course.id])) - def clean(s): - ''' Fixes URLs -- we convert spaces to _ in URLs to prevent - funny encoding characters and keep the URLs readable. This undoes - that transformation. - ''' - return s.replace('_', ' ') if s is not None else None + try: + context = { + 'csrf': csrf(request)['csrf_token'], + 'accordion': render_accordion(request, course, chapter, section), + 'COURSE_TITLE': course.title, + 'course': course, + 'init': '', + 'content': '' + } - chapter = clean(chapter) - section = clean(section) - - if settings.ENABLE_MULTICOURSE: - settings.MODULESTORE['default']['OPTIONS']['data_dir'] = settings.DATA_DIR + multicourse_settings.get_course_xmlpath(course) - - context = { - 'csrf': csrf(request)['csrf_token'], - 'accordion': render_accordion(request, course, chapter, section), - 'COURSE_TITLE': course.title, - 'course': course, - 'init': '', - 'content': '' - } - - look_for_module = chapter is not None and section is not None - if look_for_module: - # TODO (cpennington): Pass the right course in here - - section_descriptor = get_section(course, chapter, section) - if section_descriptor is not None: - student_module_cache = StudentModuleCache(request.user, - section_descriptor) - module, _, _, _ = get_module(request.user, request, - section_descriptor.location, - student_module_cache) - context['content'] = module.get_html() + look_for_module = chapter is not None and section is not None + if look_for_module: + section_descriptor = get_section(course, chapter, section) + if section_descriptor is not None: + student_module_cache = StudentModuleCache(request.user, + section_descriptor) + module, _, _, _ = get_module(request.user, request, + section_descriptor.location, + student_module_cache) + context['content'] = module.get_html() + else: + log.warning("Couldn't find a section descriptor for course_id '{0}'," + "chapter '{1}', section '{2}'".format( + course_id, chapter, section)) else: - log.warning("Couldn't find a section descriptor for course_id '{0}'," - "chapter '{1}', section '{2}'".format( - course_id, chapter, section)) + if request.user.is_staff: + # Add a list of all the errors... + context['course_errors'] = modulestore().get_item_errors(course.location) + result = render_to_response('courseware.html', context) + except: + # In production, don't want to let a 500 out for any reason + if settings.DEBUG: + raise + else: + log.exception("Error in index view: user={user}, course={course}," + " chapter={chapter} section={section}" + "position={position}".format( + user=request.user, + course=course, + chapter=chapter, + section=section, + position=position + )) + try: + result = render_to_response('courseware-error.html', {}) + except: + result = HttpResponse("There was an unrecoverable error") - result = render_to_response('courseware.html', context) return result + @ensure_csrf_cookie def jump_to(request, location): ''' @@ -237,13 +242,13 @@ def jump_to(request, location): # Complain if there's not data for this location try: - (course_id, chapter, section, position) = modulestore().path_to_location(location) + (course_id, chapter, section, position) = path_to_location(modulestore(), location) except ItemNotFoundError: raise Http404("No data at this location: {0}".format(location)) except NoPathToItem: raise Http404("This location is not in any class: {0}".format(location)) - + # Rely on index to do all error handling return index(request, course_id, chapter, section, position) @ensure_csrf_cookie @@ -258,14 +263,18 @@ def course_info(request, course_id): return render_to_response('info.html', {'course': course}) +def registered_for_course(course, user): + '''Return CourseEnrollment if user is registered for course, else False''' + if user is None: + return False + if user.is_authenticated(): + return CourseEnrollment.objects.filter(user=user, course_id=course.id).exists() + else: + return False + @ensure_csrf_cookie @cache_if_anonymous def course_about(request, course_id): - def registered_for_course(course, user): - if user.is_authenticated(): - return CourseEnrollment.objects.filter(user=user, course_id=course.id).exists() - else: - return False course = check_course(course_id, course_must_be_open=False) registered = registered_for_course(course, request.user) return render_to_response('portal/course_about.html', {'course': course, 'registered': registered}) @@ -280,7 +289,7 @@ def university_profile(request, org_id): raise Http404("University Profile not found for {0}".format(org_id)) # Only grab courses for this org... - courses = [c for c in all_courses if c.org == org_id] + courses = get_courses_by_university(request.user)[org_id] context = dict(courses=courses, org_id=org_id) template_file = "university_profile/{0}.html".format(org_id).lower() diff --git a/lms/djangoapps/dashboard/__init__.py b/lms/djangoapps/dashboard/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lms/djangoapps/dashboard/models.py b/lms/djangoapps/dashboard/models.py new file mode 100644 index 0000000000..71a8362390 --- /dev/null +++ b/lms/djangoapps/dashboard/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/lms/djangoapps/dashboard/tests.py b/lms/djangoapps/dashboard/tests.py new file mode 100644 index 0000000000..501deb776c --- /dev/null +++ b/lms/djangoapps/dashboard/tests.py @@ -0,0 +1,16 @@ +""" +This file demonstrates writing tests using the unittest module. These will pass +when you run "manage.py test". + +Replace this with more appropriate tests for your application. +""" + +from django.test import TestCase + + +class SimpleTest(TestCase): + def test_basic_addition(self): + """ + Tests that 1 + 1 always equals 2. + """ + self.assertEqual(1 + 1, 2) diff --git a/lms/djangoapps/dashboard/views.py b/lms/djangoapps/dashboard/views.py new file mode 100644 index 0000000000..c4446bceaa --- /dev/null +++ b/lms/djangoapps/dashboard/views.py @@ -0,0 +1,31 @@ +# Create your views here. +import json +from datetime import datetime +from django.http import HttpResponse, Http404 + +def dictfetchall(cursor): + '''Returns all rows from a cursor as a dict. + Borrowed from Django documentation''' + desc = cursor.description + return [ + dict(zip([col[0] for col in desc], row)) + for row in cursor.fetchall() + ] + +def dashboard(request): + """ + Quick hack to show staff enrollment numbers. This should be + replaced with a real dashboard later. This version is a short-term + bandaid for the next couple weeks. + """ + if not request.user.is_staff: + raise Http404 + + query = "select count(user_id) as students, course_id from student_courseenrollment group by course_id order by students desc" + + from django.db import connection + cursor = connection.cursor() + cursor.execute(query) + results = dictfetchall(cursor) + + return HttpResponse(json.dumps(results, indent=4)) diff --git a/lms/djangoapps/lms_migration/__init__.py b/lms/djangoapps/lms_migration/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lms/djangoapps/lms_migration/migrate.py b/lms/djangoapps/lms_migration/migrate.py new file mode 100644 index 0000000000..2bf893507b --- /dev/null +++ b/lms/djangoapps/lms_migration/migrate.py @@ -0,0 +1,110 @@ +# +# migration tools for content team to go from stable-edx4edx to LMS+CMS +# + +import logging +from pprint import pprint +import xmodule.modulestore.django as xmodule_django +from xmodule.modulestore.django import modulestore + +from django.http import HttpResponse +from django.conf import settings + +log = logging.getLogger("mitx.lms_migrate") +LOCAL_DEBUG = True +ALLOWED_IPS = settings.LMS_MIGRATION_ALLOWED_IPS + +def escape(s): + """escape HTML special characters in string""" + return str(s).replace('<','<').replace('>','>') + +def manage_modulestores(request,reload_dir=None): + ''' + Manage the static in-memory modulestores. + + If reload_dir is not None, then instruct the xml loader to reload that course directory. + ''' + html = "" + + def_ms = modulestore() + courses = def_ms.get_courses() + + #---------------------------------------- + # check on IP address of requester + + ip = request.META.get('HTTP_X_REAL_IP','') # nginx reverse proxy + if not ip: + ip = request.META.get('REMOTE_ADDR','None') + + if LOCAL_DEBUG: + html += '

      IP address: %s ' % ip + html += '

      User: %s ' % request.user + log.debug('request from ip=%s, user=%s' % (ip,request.user)) + + if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS): + if request.user and request.user.is_staff: + log.debug('request allowed because user=%s is staff' % request.user) + else: + html += 'Permission denied' + html += "" + log.debug('request denied, ALLOWED_IPS=%s' % ALLOWED_IPS) + return HttpResponse(html) + + #---------------------------------------- + # reload course if specified + + if reload_dir is not None: + if reload_dir not in def_ms.courses: + html += "

      Error: '%s' is not a valid course directory

      " % reload_dir + else: + html += "

      Reloaded course directory '%s'

      " % reload_dir + def_ms.try_load_course(reload_dir) + + #---------------------------------------- + + html += '

      Courses loaded in the modulestore

      ' + html += '
        ' + for cdir, course in def_ms.courses.items(): + html += '
      1. %s (%s)
        2. ' % (settings.MITX_ROOT_URL, + escape(cdir), + escape(cdir), + course.location.url()) + html += '
      ' + + #---------------------------------------- + + dumpfields = ['definition','location','metadata'] + + for cdir, course in def_ms.courses.items(): + html += '
      ' + html += '

      Course: %s (%s)

      ' % (course.metadata['display_name'],cdir) + + for field in dumpfields: + data = getattr(course,field) + html += '

      %s

      ' % field + if type(data)==dict: + html += '
        ' + for k,v in data.items(): + html += '
      • %s:%s
        • ' % (escape(k),escape(v)) + html += '
      ' + else: + html += '
      • %s
      ' % escape(data) + + + #---------------------------------------- + + html += '
      ' + html += "courses: 
      %s
      " % escape(courses) + + ms = xmodule_django._MODULESTORES + html += "modules: 
      %s
      " % escape(ms) + html += "default modulestore: 
      %s
      " % escape(unicode(def_ms)) + + #---------------------------------------- + + log.debug('_MODULESTORES=%s' % ms) + log.debug('courses=%s' % courses) + log.debug('def_ms=%s' % unicode(def_ms)) + + html += "" + return HttpResponse(html) diff --git a/lms/djangoapps/ssl_auth/ssl_auth.py b/lms/djangoapps/ssl_auth/ssl_auth.py deleted file mode 100755 index adbb2bf94d..0000000000 --- a/lms/djangoapps/ssl_auth/ssl_auth.py +++ /dev/null @@ -1,290 +0,0 @@ -""" -User authentication backend for ssl (no pw required) -""" - -from django.conf import settings -from django.contrib import auth -from django.contrib.auth.models import User, check_password -from django.contrib.auth.backends import ModelBackend -from django.contrib.auth.middleware import RemoteUserMiddleware -from django.core.exceptions import ImproperlyConfigured -import os -import string -import re -from random import choice - -from student.models import UserProfile - -#----------------------------------------------------------------------------- - - -def ssl_dn_extract_info(dn): - ''' - Extract username, email address (may be anyuser@anydomain.com) and full name - from the SSL DN string. Return (user,email,fullname) if successful, and None - otherwise. - ''' - ss = re.search('/emailAddress=(.*)@([^/]+)', dn) - if ss: - user = ss.group(1) - email = "%s@%s" % (user, ss.group(2)) - else: - return None - ss = re.search('/CN=([^/]+)/', dn) - if ss: - fullname = ss.group(1) - else: - return None - return (user, email, fullname) - - -def check_nginx_proxy(request): - ''' - Check for keys in the HTTP header (META) to se if we are behind an ngix reverse proxy. - If so, get user info from the SSL DN string and return that, as (user,email,fullname) - ''' - m = request.META - if m.has_key('HTTP_X_REAL_IP'): # we're behind a nginx reverse proxy, which has already done ssl auth - if not m.has_key('HTTP_SSL_CLIENT_S_DN'): - return None - dn = m['HTTP_SSL_CLIENT_S_DN'] - return ssl_dn_extract_info(dn) - return None - -#----------------------------------------------------------------------------- - - -def get_ssl_username(request): - x = check_nginx_proxy(request) - if x: - return x[0] - env = request._req.subprocess_env - if env.has_key('SSL_CLIENT_S_DN_Email'): - email = env['SSL_CLIENT_S_DN_Email'] - user = email[:email.index('@')] - return user - return None - -#----------------------------------------------------------------------------- - - -class NginxProxyHeaderMiddleware(RemoteUserMiddleware): - ''' - Django "middleware" function for extracting user information from HTTP request. - - ''' - # this field is generated by nginx's reverse proxy - header = 'HTTP_SSL_CLIENT_S_DN' # specify the request.META field to use - - def process_request(self, request): - # AuthenticationMiddleware is required so that request.user exists. - if not hasattr(request, 'user'): - raise ImproperlyConfigured( - "The Django remote user auth middleware requires the" - " authentication middleware to be installed. Edit your" - " MIDDLEWARE_CLASSES setting to insert" - " 'django.contrib.auth.middleware.AuthenticationMiddleware'" - " before the RemoteUserMiddleware class.") - - #raise ImproperlyConfigured('[ProxyHeaderMiddleware] request.META=%s' % repr(request.META)) - - try: - username = request.META[self.header] # try the nginx META key first - except KeyError: - try: - env = request._req.subprocess_env # else try the direct apache2 SSL key - if env.has_key('SSL_CLIENT_S_DN'): - username = env['SSL_CLIENT_S_DN'] - else: - raise ImproperlyConfigured('no ssl key, env=%s' % repr(env)) - username = '' - except: - # If specified header doesn't exist then return (leaving - # request.user set to AnonymousUser by the - # AuthenticationMiddleware). - return - # If the user is already authenticated and that user is the user we are - # getting passed in the headers, then the correct user is already - # persisted in the session and we don't need to continue. - - #raise ImproperlyConfigured('[ProxyHeaderMiddleware] username=%s' % username) - - if request.user.is_authenticated(): - if request.user.username == self.clean_username(username, request): - #raise ImproperlyConfigured('%s already authenticated (%s)' % (username,request.user.username)) - return - # We are seeing this user for the first time in this session, attempt - # to authenticate the user. - #raise ImproperlyConfigured('calling auth.authenticate, remote_user=%s' % username) - user = auth.authenticate(remote_user=username) - if user: - # User is valid. Set request.user and persist user in the session - # by logging the user in. - request.user = user - if settings.DEBUG: print "[ssl_auth.ssl_auth.NginxProxyHeaderMiddleware] logging in user=%s" % user - auth.login(request, user) - - def clean_username(self, username, request): - ''' - username is the SSL DN string - extract the actual username from it and return - ''' - info = ssl_dn_extract_info(username) - if not info: - return None - (username, email, fullname) = info - return username - -#----------------------------------------------------------------------------- - - -class SSLLoginBackend(ModelBackend): - ''' - Django authentication back-end which auto-logs-in a user based on having - already authenticated with an MIT certificate (SSL). - ''' - def authenticate(self, username=None, password=None, remote_user=None): - - # remote_user is from the SSL_DN string. It will be non-empty only when - # the user has already passed the server authentication, which means - # matching with the certificate authority. - if not remote_user: - # no remote_user, so check username (but don't auto-create user) - if not username: - return None - return None # pass on to another authenticator backend - #raise ImproperlyConfigured("in SSLLoginBackend, username=%s, remote_user=%s" % (username,remote_user)) - try: - user = User.objects.get(username=username) # if user already exists don't create it - return user - except User.DoesNotExist: - return None - return None - - #raise ImproperlyConfigured("in SSLLoginBackend, username=%s, remote_user=%s" % (username,remote_user)) - #if not os.environ.has_key('HTTPS'): - # return None - #if not os.environ.get('HTTPS')=='on': # only use this back-end if HTTPS on - # return None - - def GenPasswd(length=8, chars=string.letters + string.digits): - return ''.join([choice(chars) for i in range(length)]) - - # convert remote_user to user, email, fullname - info = ssl_dn_extract_info(remote_user) - #raise ImproperlyConfigured("[SSLLoginBackend] looking up %s" % repr(info)) - if not info: - #raise ImproperlyConfigured("[SSLLoginBackend] remote_user=%s, info=%s" % (remote_user,info)) - return None - (username, email, fullname) = info - - try: - user = User.objects.get(username=username) # if user already exists don't create it - except User.DoesNotExist: - if not settings.DEBUG: - raise "User does not exist. Not creating user; potential schema consistency issues" - #raise ImproperlyConfigured("[SSLLoginBackend] creating %s" % repr(info)) - user = User(username=username, password=GenPasswd()) # create new User - user.is_staff = False - user.is_superuser = False - # get first, last name from fullname - name = fullname - if not name.count(' '): - user.first_name = " " - user.last_name = name - mn = '' - else: - user.first_name = name[:name.find(' ')] - ml = name[name.find(' '):].strip() - if ml.count(' '): - user.last_name = ml[ml.rfind(' '):] - mn = ml[:ml.rfind(' ')] - else: - user.last_name = ml - mn = '' - # set email - user.email = email - # cleanup last name - user.last_name = user.last_name.strip() - # save - user.save() - - # auto-create user profile - up = UserProfile(user=user) - up.name = fullname - up.save() - - #tui = user.get_profile() - #tui.middle_name = mn - #tui.role = 'Misc' - #tui.section = None # no section assigned at first - #tui.save() - # return None - return user - - def get_user(self, user_id): - #if not os.environ.has_key('HTTPS'): - # return None - #if not os.environ.get('HTTPS')=='on': # only use this back-end if HTTPS on - # return None - try: - return User.objects.get(pk=user_id) - except User.DoesNotExist: - return None - -#----------------------------------------------------------------------------- -# OLD! - - -class AutoLoginBackend: - def authenticate(self, username=None, password=None): - raise ImproperlyConfigured("in AutoLoginBackend, username=%s" % username) - if not os.environ.has_key('HTTPS'): - return None - if not os.environ.get('HTTPS') == 'on':# only use this back-end if HTTPS on - return None - - def GenPasswd(length=8, chars=string.letters + string.digits): - return ''.join([choice(chars) for i in range(length)]) - - try: - user = User.objects.get(username=username) - except User.DoesNotExist: - user = User(username=username, password=GenPasswd()) - user.is_staff = False - user.is_superuser = False - # get first, last name - name = os.environ.get('SSL_CLIENT_S_DN_CN').strip() - if not name.count(' '): - user.first_name = " " - user.last_name = name - mn = '' - else: - user.first_name = name[:name.find(' ')] - ml = name[name.find(' '):].strip() - if ml.count(' '): - user.last_name = ml[ml.rfind(' '):] - mn = ml[:ml.rfind(' ')] - else: - user.last_name = ml - mn = '' - # get email - user.email = os.environ.get('SSL_CLIENT_S_DN_Email') - # save - user.save() - tui = user.get_profile() - tui.middle_name = mn - tui.role = 'Misc' - tui.section = None# no section assigned at first - tui.save() - # return None - return user - - def get_user(self, user_id): - if not os.environ.has_key('HTTPS'): - return None - if not os.environ.get('HTTPS') == 'on':# only use this back-end if HTTPS on - return None - try: - return User.objects.get(pk=user_id) - except User.DoesNotExist: - return None diff --git a/lms/envs/common.py b/lms/envs/common.py index 032a088195..83a4bd4181 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -48,6 +48,17 @@ MITX_FEATURES = { ## DO NOT SET TO True IN THIS FILE ## Doing so will cause all courses to be released on production 'DISABLE_START_DATES': False, # When True, all courses will be active, regardless of start date + + 'ENABLE_TEXTBOOK' : True, + 'ENABLE_DISCUSSION' : True, + + 'ENABLE_SQL_TRACKING_LOGS': False, + 'ENABLE_LMS_MIGRATION': False, + + # extrernal access methods + 'ACCESS_REQUIRE_STAFF_FOR_COURSE': False, + 'AUTH_USE_OPENID': False, + 'AUTH_USE_MIT_CERTIFICATES' : False, } # Used for A/B testing @@ -304,7 +315,7 @@ PIPELINE_CSS = { 'output_filename': 'css/lms-application.css', }, 'course': { - 'source_filenames': ['sass/course.scss', 'js/vendor/CodeMirror/codemirror.css', 'css/vendor/jquery.treeview.css', 'css/vendor/jquery-ui-1.8.22.custom.css', 'css/vendor/jquery.qtip.min.css'], + 'source_filenames': ['js/vendor/CodeMirror/codemirror.css', 'css/vendor/jquery.treeview.css', 'css/vendor/ui-lightness/jquery-ui-1.8.22.custom.css', 'css/vendor/jquery.qtip.min.css', 'sass/course.scss'], 'output_filename': 'css/lms-course.css', }, 'ie-fixes': { diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 1a2659cb1f..bc5b621b32 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -14,10 +14,11 @@ DEBUG = True TEMPLATE_DEBUG = True MITX_FEATURES['DISABLE_START_DATES'] = True +MITX_FEATURES['ENABLE_SQL_TRACKING_LOGS'] = True WIKI_ENABLED = True -LOGGING = get_logger_config(ENV_ROOT / "log", +LOGGING = get_logger_config(ENV_ROOT / "log", logging_env="dev", tracking_filename="tracking.log", debug=True) @@ -30,7 +31,7 @@ DATABASES = { } CACHES = { - # This is the cache used for most things. Askbot will not work without a + # This is the cache used for most things. Askbot will not work without a # functioning cache -- it relies on caching to load its settings in places. # In staging/prod envs, the sessions also live here. 'default': { @@ -52,11 +53,35 @@ CACHES = { } } +# Make the keyedcache startup warnings go away +CACHE_TIMEOUT = 0 + # Dummy secret key for dev SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' +################################ LMS Migration ################################# +MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True +MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = False # require that user be in the staff_* group to be able to enroll + +LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1'] + +################################ OpenID Auth ################################# +MITX_FEATURES['AUTH_USE_OPENID'] = True +MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True + +INSTALLED_APPS += ('external_auth',) +INSTALLED_APPS += ('django_openid_auth',) + +OPENID_CREATE_USERS = False +OPENID_UPDATE_DETAILS_FROM_SREG = True +OPENID_SSO_SERVER_URL = 'https://www.google.com/accounts/o8/id' # TODO: accept more endpoints +OPENID_USE_AS_ADMIN_LOGIN = False + +################################ MIT Certificates SSL Auth ################################# +MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True + ################################ DEBUG TOOLBAR ################################# -INSTALLED_APPS += ('debug_toolbar',) +INSTALLED_APPS += ('debug_toolbar',) MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',) INTERNAL_IPS = ('127.0.0.1',) @@ -71,8 +96,8 @@ DEBUG_TOOLBAR_PANELS = ( 'debug_toolbar.panels.logger.LoggingPanel', # Enabling the profiler has a weird bug as of django-debug-toolbar==0.9.4 and -# Django=1.3.1/1.4 where requests to views get duplicated (your method gets -# hit twice). So you can uncomment when you need to diagnose performance +# Django=1.3.1/1.4 where requests to views get duplicated (your method gets +# hit twice). So you can uncomment when you need to diagnose performance # problems, but you shouldn't leave it on. # 'debug_toolbar.panels.profiling.ProfilingDebugPanel', ) diff --git a/lms/envs/dev_ike.py b/lms/envs/dev_ike.py index 79ae3354ac..2256decb46 100644 --- a/lms/envs/dev_ike.py +++ b/lms/envs/dev_ike.py @@ -7,142 +7,12 @@ sessions. Assumes structure: /mitx # The location of this repo /log # Where we're going to write log files """ - -import socket - -if 'eecs1' in socket.gethostname(): - MITX_ROOT_URL = '/mitx2' - from .common import * from .logsettings import get_logger_config from .dev import * -if 'eecs1' in socket.gethostname(): - # MITX_ROOT_URL = '/mitx2' - MITX_ROOT_URL = 'https://eecs1.mit.edu/mitx2' +WIKI_ENABLED = False +MITX_FEATURES['ENABLE_TEXTBOOK'] = False +MITX_FEATURES['ENABLE_DISCUSSION'] = False +MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True # require that user be in the staff_* group to be able to enroll -#----------------------------------------------------------------------------- -# edx4edx content server - -EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' -MITX_FEATURES['REROUTE_ACTIVATION_EMAIL'] = 'ichuang@mit.edu' -EDX4EDX_ROOT = ENV_ROOT / "data/edx4edx" - -#EMAIL_BACKEND = 'django_ses.SESBackend' - -#----------------------------------------------------------------------------- -# ichuang - -DEBUG = True -ENABLE_MULTICOURSE = True # set to False to disable multicourse display (see lib.util.views.mitxhome) -QUICKEDIT = False - -MAKO_TEMPLATES['course'] = [DATA_DIR, EDX4EDX_ROOT ] - -#MITX_FEATURES['USE_DJANGO_PIPELINE'] = False -MITX_FEATURES['DISPLAY_HISTOGRAMS_TO_STAFF'] = False -MITX_FEATURES['DISPLAY_EDIT_LINK'] = True -MITX_FEATURES['DEBUG_LEVEL'] = 10 # 0 = lowest level, least verbose, 255 = max level, most verbose - -COURSE_SETTINGS = {'6.002x_Fall_2012': {'number' : '6.002x', - 'title' : 'Circuits and Electronics', - 'xmlpath': '/6002x-fall-2012/', - 'active' : True, - 'default_chapter' : 'Week_1', - 'default_section' : 'Administrivia_and_Circuit_Elements', - 'location': 'i4x://edx/6002xs12/course/6.002x_Fall_2012', - }, - '8.02_Spring_2013': {'number' : '8.02x', - 'title' : 'Electricity & Magnetism', - 'xmlpath': '/802x/', - 'github_url': 'https://github.com/MITx/8.02x', - 'active' : True, - 'default_chapter' : 'Introduction', - 'default_section' : 'Introduction_%28Lewin_2002%29', - }, - '6.189_Spring_2013': {'number' : '6.189x', - 'title' : 'IAP Python Programming', - 'xmlpath': '/6.189x/', - 'github_url': 'https://github.com/MITx/6.189x', - 'active' : True, - 'default_chapter' : 'Week_1', - 'default_section' : 'Variables_and_Binding', - }, - '8.01_Fall_2012': {'number' : '8.01x', - 'title' : 'Mechanics', - 'xmlpath': '/8.01x/', - 'github_url': 'https://github.com/MITx/8.01x', - 'active': True, - 'default_chapter' : 'Mechanics_Online_Spring_2012', - 'default_section' : 'Introduction_to_the_course', - 'location': 'i4x://edx/6002xs12/course/8.01_Fall_2012', - }, - 'edx4edx': {'number' : 'edX.01', - 'title' : 'edx4edx: edX Author Course', - 'xmlpath': '/edx4edx/', - 'github_url': 'https://github.com/MITx/edx4edx', - 'active' : True, - 'default_chapter' : 'Introduction', - 'default_section' : 'edx4edx_Course', - 'location': 'i4x://edx/6002xs12/course/edx4edx', - }, - '7.03x_Fall_2012': {'number' : '7.03x', - 'title' : 'Genetics', - 'xmlpath': '/7.03x/', - 'github_url': 'https://github.com/MITx/7.03x', - 'active' : True, - 'default_chapter' : 'Week_2', - 'default_section' : 'ps1_question_1', - }, - '3.091x_Fall_2012': {'number' : '3.091x', - 'title' : 'Introduction to Solid State Chemistry', - 'xmlpath': '/3.091x/', - 'github_url': 'https://github.com/MITx/3.091x', - 'active' : True, - 'default_chapter' : 'Week_1', - 'default_section' : 'Problem_Set_1', - }, - '18.06x_Linear_Algebra': {'number' : '18.06x', - 'title' : 'Linear Algebra', - 'xmlpath': '/18.06x/', - 'github_url': 'https://github.com/MITx/18.06x', - 'default_chapter' : 'Unit_1', - 'default_section' : 'Midterm_1', - 'active' : True, - }, - '6.00x_Fall_2012': {'number' : '6.00x', - 'title' : 'Introduction to Computer Science and Programming', - 'xmlpath': '/6.00x/', - 'github_url': 'https://github.com/MITx/6.00x', - 'active' : True, - 'default_chapter' : 'Week_0', - 'default_section' : 'Problem_Set_0', - 'location': 'i4x://edx/6002xs12/course/6.00x_Fall_2012', - }, - '7.00x_Fall_2012': {'number' : '7.00x', - 'title' : 'Introduction to Biology', - 'xmlpath': '/7.00x/', - 'github_url': 'https://github.com/MITx/7.00x', - 'active' : True, - 'default_chapter' : 'Unit 1', - 'default_section' : 'Introduction', - }, - } - -#----------------------------------------------------------------------------- - -MIDDLEWARE_CLASSES = MIDDLEWARE_CLASSES + ( - 'ssl_auth.ssl_auth.NginxProxyHeaderMiddleware', # ssl authentication behind nginx proxy - ) - -AUTHENTICATION_BACKENDS = ( - 'ssl_auth.ssl_auth.SSLLoginBackend', - 'django.contrib.auth.backends.ModelBackend', - ) - -INSTALLED_APPS = INSTALLED_APPS + ( - 'ssl_auth', - ) - -LOGIN_REDIRECT_URL = MITX_ROOT_URL + '/' -LOGIN_URL = MITX_ROOT_URL + '/' diff --git a/lms/static/sass/application.scss b/lms/static/sass/application.scss index c16d72e367..240d68c5b3 100644 --- a/lms/static/sass/application.scss +++ b/lms/static/sass/application.scss @@ -2,9 +2,9 @@ @import 'base/reset'; @import 'base/font_face'; +@import 'base/mixins'; @import 'base/variables'; @import 'base/base'; -@import 'base/mixins'; @import 'base/extends'; @import 'base/animations'; diff --git a/lms/static/sass/base/_mixins.scss b/lms/static/sass/base/_mixins.scss index 7c53b6e14f..58a92d1ee6 100644 --- a/lms/static/sass/base/_mixins.scss +++ b/lms/static/sass/base/_mixins.scss @@ -1,3 +1,7 @@ +@function em($pxval, $base: 16) { + @return #{$pxval / $base}em; +} + // Line-height @function lh($amount: 1) { @return $body-line-height * $amount; diff --git a/lms/static/sass/base/_variables.scss b/lms/static/sass/base/_variables.scss index 6c8d0d4000..7ad30f0c91 100644 --- a/lms/static/sass/base/_variables.scss +++ b/lms/static/sass/base/_variables.scss @@ -4,10 +4,15 @@ $gw-gutter: 20px; $fg-column: $gw-column; $fg-gutter: $gw-gutter; $fg-max-columns: 12; +$fg-max-width: 1400px; +$fg-min-width: 810px; $sans-serif: 'Open Sans', $verdana; +$body-font-family: $sans-serif; $serif: $georgia; +$body-font-size: em(14); +$body-line-height: golden-ratio(.875em, 1); $base-font-color: rgb(60,60,60); $lighter-base-font-color: rgb(160,160,160); @@ -15,18 +20,11 @@ $blue: rgb(29,157,217); $pink: rgb(182,37,104); $yellow: rgb(255, 252, 221); $error-red: rgb(253, 87, 87); +$border-color: #C8C8C8; // old variables -$body-font-family: "Open Sans", "Lucida Grande", "Lucida Sans Unicode", "Lucida Sans", Geneva, Verdana, sans-serif; -$body-font-size: 14px; -$body-line-height: golden-ratio($body-font-size, 1); - -$fg-max-width: 1400px; -$fg-min-width: 810px; $light-gray: #ddd; $dark-gray: #333; $mit-red: #993333; -$cream: #F6EFD4; $text-color: $dark-gray; -$border-color: $light-gray; diff --git a/lms/static/sass/course.scss b/lms/static/sass/course.scss index 8fafbb8479..cc1b49a0a2 100644 --- a/lms/static/sass/course.scss +++ b/lms/static/sass/course.scss @@ -2,9 +2,9 @@ @import 'base/reset'; @import 'base/font_face'; +@import 'base/mixins'; @import 'base/variables'; @import 'base/base'; -@import 'base/mixins'; @import 'base/extends'; @import 'base/animations'; diff --git a/lms/static/sass/course/_help.scss b/lms/static/sass/course/_help.scss deleted file mode 100644 index cb505814e9..0000000000 --- a/lms/static/sass/course/_help.scss +++ /dev/null @@ -1,54 +0,0 @@ -section.help.main-content { - padding: lh(); - - h1 { - border-bottom: 1px solid #ddd; - margin-bottom: lh(); - margin-top: 0; - padding-bottom: lh(); - } - - p { - max-width: 700px; - } - - h2 { - margin-top: 0; - } - - section.self-help { - float: left; - margin-bottom: lh(); - margin-right: flex-gutter(); - width: flex-grid(6); - - ul { - margin-left: flex-gutter(6); - - li { - margin-bottom: lh(.5); - } - } - } - - section.help-email { - float: left; - width: flex-grid(6); - - dl { - display: block; - margin-bottom: lh(); - - dd { - margin-bottom: lh(); - } - - dt { - clear: left; - float: left; - font-weight: bold; - width: flex-grid(2, 6); - } - } - } -} diff --git a/lms/static/sass/course/_info.scss b/lms/static/sass/course/_info.scss index 45dd2d57b3..d9af9c0f82 100644 --- a/lms/static/sass/course/_info.scss +++ b/lms/static/sass/course/_info.scss @@ -3,6 +3,7 @@ div.info-wrapper { section.updates { @extend .content; + line-height: lh(); > h1 { @extend .top-header; @@ -15,30 +16,35 @@ div.info-wrapper { > ol { list-style: none; padding-left: 0; + margin-bottom: lh(); > li { @extend .clearfix; border-bottom: 1px solid #e3e3e3; - margin-bottom: lh(.5); + margin-bottom: lh(); padding-bottom: lh(.5); list-style-type: disk; &:first-child { - background: $cream; - border-bottom: 1px solid darken($cream, 10%); margin: 0 (-(lh(.5))) lh(); padding: lh(.5); } ol, ul { - margin: lh() 0 0 lh(); - list-style-type: circle; + margin: 0; + list-style-type: disk; + + ol,ul { + list-style-type: circle; + } } h2 { float: left; margin: 0 flex-gutter() 0 0; width: flex-grid(2, 9); + font-size: $body-font-size; + font-weight: bold; } section.update-description { @@ -64,16 +70,20 @@ div.info-wrapper { @extend .sidebar; border-left: 1px solid #d3d3d3; @include border-radius(0 4px 4px 0); + @include box-shadow(none); border-right: 0; - header { + h1 { @extend .bottom-border; - padding: lh(.5) lh(.75); + padding: lh(.5) lh(.5); + } - h1 { - font-size: 18px; - margin: 0 ; - } + header { + + // h1 { + // font-weight: 100; + // font-style: italic; + // } p { color: #666; @@ -94,7 +104,7 @@ div.info-wrapper { border-bottom: 1px solid #d3d3d3; @include box-shadow(0 1px 0 #eee); @include box-sizing(border-box); - padding: 7px lh(.75); + padding: em(7) lh(.75); position: relative; &.expandable, @@ -108,13 +118,13 @@ div.info-wrapper { ul { background: none; - margin: 7px (-(lh(.75))) 0; + margin: em(7) (-(lh(.75))) 0; li { border-bottom: 0; border-top: 1px solid #d3d3d3; @include box-shadow(inset 0 1px 0 #eee); - padding-left: 18px + lh(.75); + padding-left: lh(1.5); } } @@ -150,7 +160,7 @@ div.info-wrapper { border-bottom: 0; @include box-shadow(none); color: #999; - font-size: 12px; + font-size: $body-font-size; font-weight: bold; text-transform: uppercase; } diff --git a/lms/static/sass/course/_textbook.scss b/lms/static/sass/course/_textbook.scss index ae549d723f..ed5e528809 100644 --- a/lms/static/sass/course/_textbook.scss +++ b/lms/static/sass/course/_textbook.scss @@ -62,7 +62,6 @@ div.book-wrapper { @extend .clearfix; li { - background-color: darken($cream, 4%); &.last { display: block; diff --git a/lms/static/sass/course/base/_base.scss b/lms/static/sass/course/base/_base.scss index 02aba1866e..d04bcab103 100644 --- a/lms/static/sass/course/base/_base.scss +++ b/lms/static/sass/course/base/_base.scss @@ -5,3 +5,17 @@ body { h1, h2, h3, h4, h5, h6 { font-family: $sans-serif; } + +table { + table-layout: fixed; +} + +.container { + padding: lh(2); + + > div { + display: table; + width: 100%; + table-layout: fixed; + } +} diff --git a/lms/static/sass/course/base/_extends.scss b/lms/static/sass/course/base/_extends.scss index b71b8161f6..5927cb569e 100644 --- a/lms/static/sass/course/base/_extends.scss +++ b/lms/static/sass/course/base/_extends.scss @@ -1,43 +1,9 @@ -.wrapper { - margin: 0 auto; - max-width: $fg-max-width; - min-width: $fg-min-width; - text-align: left; - width: flex-grid(12); - - div.table-wrapper { - display: table; - width: flex-grid(12); - overflow: hidden; - } -} - h1.top-header { - background: #f3f3f3; border-bottom: 1px solid #e3e3e3; - margin: (-(lh())) (-(lh())) lh(); - padding: lh(); text-align: left; -} - -.button { - border: 1px solid darken(#888, 10%); - @include border-radius(3px); - @include box-shadow(inset 0 1px 0 lighten(#888, 10%), 0 0 3px #ccc); - color: #fff; - cursor: pointer; - font: bold $body-font-size $body-font-family; - @include linear-gradient(lighten(#888, 5%), darken(#888, 5%)); - padding: 4px 8px; - text-decoration: none; - text-shadow: none; - -webkit-font-smoothing: antialiased; - - &:hover, &:focus { - border: 1px solid darken(#888, 20%); - @include box-shadow(inset 0 1px 0 lighten(#888, 20%), 0 0 3px #ccc); - @include linear-gradient(lighten(#888, 10%), darken(#888, 5%)); - } + font-size: 24px; + font-weight: 100; + padding-bottom: lh(); } .light-button, a.light-button { @@ -84,7 +50,8 @@ h1.top-header { } .sidebar { - border-right: 1px solid #d3d3d3; + border-right: 1px solid #C8C8C8; + @include box-shadow(inset -1px 0 0 #e6e6e6); @include box-sizing(border-box); display: table-cell; font-family: $sans-serif; @@ -93,11 +60,13 @@ h1.top-header { width: flex-grid(3); h1, h2 { - font-size: 18px; - font-weight: bold; + font-size: em(18); + font-weight: 100; letter-spacing: 0; text-transform: none; font-family: $sans-serif; + text-align: left; + font-style: italic; } a { @@ -146,27 +115,20 @@ h1.top-header { } header#open_close_accordion { - border-bottom: 1px solid #d3d3d3; - @include box-shadow(0 1px 0 #eee); - padding: lh(.5) lh(); position: relative; - h2 { - margin: 0; - padding-right: 20px; - } - a { - background: #eee url('../images/slide-left-icon.png') center center no-repeat; + background: #f6f6f6 url('../images/slide-left-icon.png') center center no-repeat; border: 1px solid #D3D3D3; @include border-radius(3px 0 0 3px); height: 16px; - padding: 8px; + padding: 6px; position: absolute; right: -1px; text-indent: -9999px; top: 6px; width: 16px; + z-index: 99; &:hover { background-color: white; @@ -181,33 +143,17 @@ h1.top-header { .topbar { @extend .clearfix; - background: $cream; - border-bottom: 1px solid darken($cream, 10%); - border-top: 1px solid #fff; - font-size: 12px; - line-height: 46px; - text-shadow: 0 1px 0 #fff; + border-bottom: 1px solid $border-color; + font-size: 14px; @media print { display: none; } a { - line-height: 46px; - border-bottom: 0; - color: darken($cream, 80%); - - &:hover { - color: darken($cream, 60%); - text-decoration: none; - } - &.block-link { - // background: darken($cream, 5%); - border-left: 1px solid darken($cream, 20%); - @include box-shadow(inset 1px 0 0 lighten($cream, 5%)); + border-left: 1px solid lighten($border-color, 10%); display: block; - text-transform: uppercase; &:hover { background: none; @@ -219,12 +165,3 @@ h1.top-header { .tran { @include transition( all, .2s, $ease-in-out-quad); } - -p.ie-warning { - background: yellow; - display: block !important; - line-height: 1.3em; - margin-bottom: 0; - padding: lh(); - text-align: left; -} diff --git a/lms/static/sass/course/courseware/_courseware.scss b/lms/static/sass/course/courseware/_courseware.scss index 8a0d880ceb..f6c9dceb8e 100644 --- a/lms/static/sass/course/courseware/_courseware.scss +++ b/lms/static/sass/course/courseware/_courseware.scss @@ -3,22 +3,6 @@ html { max-height: 100%; } -body.courseware { - height: 100%; - max-height: 100%; - - .container { - padding-bottom: 40px; - margin-top: 20px; - } - - footer { - &.fixed-bottom { - Position: static; - } - } -} - div.course-wrapper { @extend .table-wrapper; @@ -59,6 +43,9 @@ div.course-wrapper { } ol.vert-mod { + padding: 0; + margin: 0; + > li { @extend .clearfix; @extend .problem-set; @@ -194,17 +181,9 @@ div.course-wrapper { overflow: hidden; header#open_close_accordion { - padding: 0; - min-height: 47px; - a { background-image: url('../images/slide-right-icon.png'); } - - h2 { - visibility: hidden; - width: 10px; - } } div#accordion { diff --git a/lms/static/sass/course/courseware/_sidebar.scss b/lms/static/sass/course/courseware/_sidebar.scss index fc0291f48a..fe9f54d0e3 100644 --- a/lms/static/sass/course/courseware/_sidebar.scss +++ b/lms/static/sass/course/courseware/_sidebar.scss @@ -13,44 +13,51 @@ section.course-index { div#accordion { h3 { - @include box-shadow(inset 0 1px 0 0 #eee); - border-top: 1px solid #d3d3d3; - overflow: hidden; + @include border-radius(0); + border-top: 1px solid #e3e3e3; margin: 0; + overflow: hidden; &:first-child { border: none; } &:hover { - @include background-image(linear-gradient(-90deg, rgb(245,245,245), rgb(225,225,225))); + background: #f6f6f6; + text-decoration: none; } &.ui-accordion-header { color: #000; a { - font-size: $body-font-size; + @include border-radius(0); + @include box-shadow(none); color: lighten($text-color, 10%); + font-size: $body-font-size; } &.ui-state-active { - @include background-image(linear-gradient(-90deg, rgb(245,245,245), rgb(225,225,225))); @extend .active; - border-bottom: 1px solid #d3d3d3; + border-bottom: none; + + &:hover { + background: none; + } } } } ul.ui-accordion-content { @include border-radius(0); - @include box-shadow(inset -1px 0 0 #e6e6e6); + background: transparent; border: none; font-size: 12px; margin: 0; padding: 1em 1.5em; li { + @include border-radius(0); margin-bottom: lh(.5); a { @@ -98,7 +105,7 @@ section.course-index { &:after { opacity: 1; right: 15px; - @include transition(all, 0.2s, linear); + @include transition(); } > a p { @@ -120,8 +127,6 @@ section.course-index { font-weight: bold; > a { - background: rgb(240,240,240); - @include background-image(linear-gradient(-90deg, rgb(245,245,245), rgb(230,230,230))); border-color: rgb(200,200,200); &:after { diff --git a/lms/static/sass/course/discussion/_discussion.scss b/lms/static/sass/course/discussion/_discussion.scss index b9022a43d8..7b0aa601d9 100644 --- a/lms/static/sass/course/discussion/_discussion.scss +++ b/lms/static/sass/course/discussion/_discussion.scss @@ -1,8 +1,6 @@ // Generic layout styles for the discussion forums - body.askbot { - - section.main-content { + section.container { div.discussion-wrapper { @extend .table-wrapper; diff --git a/lms/static/sass/course/discussion/_profile.scss b/lms/static/sass/course/discussion/_profile.scss index 42e6b772f8..010a03ffd6 100644 --- a/lms/static/sass/course/discussion/_profile.scss +++ b/lms/static/sass/course/discussion/_profile.scss @@ -72,7 +72,6 @@ body.user-profile-page { margin-bottom: 30px; li { - background-color: lighten($cream, 3%); background-position: 10px center; background-repeat: no-repeat; @include border-radius(4px); diff --git a/lms/static/sass/course/discussion/_question-view.scss b/lms/static/sass/course/discussion/_question-view.scss index 4b7765b2f9..4c2acaf9be 100644 --- a/lms/static/sass/course/discussion/_question-view.scss +++ b/lms/static/sass/course/discussion/_question-view.scss @@ -32,8 +32,6 @@ div.question-header { &.post-vote { @include border-radius(4px); - background-color: lighten($cream, 5%); - border: 1px solid darken( $cream, 10% ); @include box-shadow(inset 0 1px 0px #fff); } @@ -149,7 +147,7 @@ div.question-header { &.revision { text-align: center; - background:lighten($cream, 7%); + // background:lighten($cream, 7%); a { color: black; @@ -313,7 +311,6 @@ div.question-header { } a.edit { - @extend .button; font-size: 12px; padding: 2px 10px; } diff --git a/lms/static/sass/course/layout/_courseware_subnav.scss b/lms/static/sass/course/layout/_courseware_subnav.scss index 7d2433138e..21f6187a83 100644 --- a/lms/static/sass/course/layout/_courseware_subnav.scss +++ b/lms/static/sass/course/layout/_courseware_subnav.scss @@ -1,9 +1,8 @@ nav.course-material { - background: rgb(210,210,210); @include clearfix; @include box-sizing(border-box); - @include box-shadow(inset 0 1px 5px 0 rgba(0,0,0, 0.05)); - border-bottom: 1px solid rgb(190,190,190); + background: #f6f6f6; + border-bottom: 1px solid rgb(200,200,200); margin: 0px auto 0px; padding: 0px; width: 100%; @@ -24,12 +23,14 @@ nav.course-material { list-style: none; a { - color: $lighter-base-font-color; + color: darken($lighter-base-font-color, 20%); display: block; text-align: center; - padding: 5px 13px; + padding: 8px 13px 12px; + font-size: 14px; + font-weight: 400; text-decoration: none; - text-shadow: 0 1px rgba(255,255,255, 0.4); + text-shadow: 0 1px rgb(255,255,255); &:hover { color: $base-font-color; @@ -41,7 +42,7 @@ nav.course-material { border-bottom: 0px; @include border-top-radius(4px); @include box-shadow(0 2px 0 0 rgba(255,255,255, 1)); - color: $base-font-color; + color: $blue; } } } diff --git a/lms/static/sass/course/old/.gitignore b/lms/static/sass/course/old/.gitignore deleted file mode 100644 index b3a5267117..0000000000 --- a/lms/static/sass/course/old/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.css diff --git a/lms/static/sass/course/wiki/_wiki.scss b/lms/static/sass/course/wiki/_wiki.scss index 9c878ad263..ec53044ed1 100644 --- a/lms/static/sass/course/wiki/_wiki.scss +++ b/lms/static/sass/course/wiki/_wiki.scss @@ -24,7 +24,6 @@ div.wiki-wrapper { } p { - color: darken($cream, 55%); float: left; line-height: 46px; margin-bottom: 0; @@ -40,14 +39,12 @@ div.wiki-wrapper { input[type="button"] { @extend .block-link; - background-color: darken($cream, 5%); background-position: 12px center; background-repeat: no-repeat; border: 0; border-left: 1px solid darken(#f6efd4, 20%); @include border-radius(0); @include box-shadow(inset 1px 0 0 lighten(#f6efd4, 5%)); - color: darken($cream, 80%); display: block; font-size: 12px; font-weight: normal; diff --git a/lms/static/sass/multicourse/_dashboard.scss b/lms/static/sass/multicourse/_dashboard.scss index a3d21cb1b3..9c2b71f5c0 100644 --- a/lms/static/sass/multicourse/_dashboard.scss +++ b/lms/static/sass/multicourse/_dashboard.scss @@ -2,6 +2,27 @@ @include clearfix; padding: 60px 0px 120px; + .dashboard-banner { + background: $yellow; + border: 1px solid rgb(200,200,200); + @include box-shadow(0 1px 0 0 rgba(255,255,255, 0.6)); + padding: 10px; + margin-bottom: 30px; + + &:empty { + display: none; + background-color: #FFF; + } + + h2 { + margin-bottom: 0; + } + + p { + margin-bottom: 0; + } + } + .profile-sidebar { background: transparent; float: left; diff --git a/lms/templates/accordion.html b/lms/templates/accordion.html index defb424a29..353b83db70 100644 --- a/lms/templates/accordion.html +++ b/lms/templates/accordion.html @@ -1,13 +1,13 @@ <%! from django.core.urlresolvers import reverse %> <%def name="make_chapter(chapter)"> -

      ${chapter['name']}

      +

      ${chapter['display_name']}

    • diff --git a/lms/templates/dashboard.html b/lms/templates/dashboard.html index ca3b273fc3..480568a5b9 100644 --- a/lms/templates/dashboard.html +++ b/lms/templates/dashboard.html @@ -34,10 +34,11 @@
    -
    - ${message} -
    -
    + %if message: +
    + ${message} +
    + %endif
    diff --git a/lms/templates/extauth_failure.html b/lms/templates/extauth_failure.html new file mode 100644 index 0000000000..fa53ab1084 --- /dev/null +++ b/lms/templates/extauth_failure.html @@ -0,0 +1,11 @@ + + + + OpenID failed + + +

    OpenID failed

    +

    ${message}

    + + diff --git a/lms/templates/index.html b/lms/templates/index.html index d8b0394927..4255251604 100644 --- a/lms/templates/index.html +++ b/lms/templates/index.html @@ -144,3 +144,31 @@
    + +% if show_signup_immediately is not UNDEFINED: + +% endif diff --git a/lms/templates/info.html b/lms/templates/info.html index 25ad4f9184..a04e31896f 100644 --- a/lms/templates/info.html +++ b/lms/templates/info.html @@ -20,23 +20,25 @@ $(document).ready(function(){
    -
    -
    - % if user.is_authenticated(): -
    - ${get_course_info_section(course, 'updates')} -
    -
    - ${get_course_info_section(course, 'handouts')} -
    - % else: -
    - ${get_course_info_section(course, 'guest_updates')} -
    -
    - ${get_course_info_section(course, 'guest_handouts')} -
    - % endif -
    -
    +
    + % if user.is_authenticated(): +
    +

    Course Updates & News

    + ${get_course_info_section(course, 'updates')} +
    +
    +

    Course Handouts

    + ${get_course_info_section(course, 'handouts')} +
    + % else: +
    +

    Course Updates & News

    + ${get_course_info_section(course, 'guest_updates')} +
    +
    +

    Course Handouts

    + ${get_course_info_section(course, 'guest_handouts')} +
    + % endif +
    diff --git a/lms/templates/login_modal.html b/lms/templates/login_modal.html index 393e76ee78..c89931695c 100644 --- a/lms/templates/login_modal.html +++ b/lms/templates/login_modal.html @@ -27,6 +27,9 @@ Not enrolled? Forgot password?

    +

    + login via openid +

    diff --git a/lms/templates/main.html b/lms/templates/main.html index 7ef8960970..fb502bfe22 100644 --- a/lms/templates/main.html +++ b/lms/templates/main.html @@ -3,7 +3,7 @@ <%block name="title">edX - + <%static:css group='application'/> diff --git a/lms/templates/module-error.html b/lms/templates/module-error.html index 28597fa13c..7c731db17a 100644 --- a/lms/templates/module-error.html +++ b/lms/templates/module-error.html @@ -1,4 +1,13 @@

    There has been an error on the MITx servers

    We're sorry, this module is temporarily unavailable. Our staff is working to fix it as soon as possible. Please email us at technical@mitx.mit.edu to report any problems or downtime.

    + +% if is_staff: +

    Staff-only details below:

    + +

    Error: ${error | h}

    + +

    Raw data: ${data | h}

    +% endif +
    diff --git a/lms/templates/portal/course_about.html b/lms/templates/portal/course_about.html index e6359d0542..a3bf8dd755 100644 --- a/lms/templates/portal/course_about.html +++ b/lms/templates/portal/course_about.html @@ -19,6 +19,8 @@ $(document).delegate('#class_enroll_form', 'ajax:success', function(data, json, xhr) { if(json.success) { location.href="${reverse('dashboard')}"; + }else{ + $('#register_message).html("

    " + json.error + "

    ") } }); })(this) @@ -60,9 +62,24 @@
    %if user.is_authenticated(): %if registered: + <% + ## TODO: move this logic into a view + if course.has_started() or settings.MITX_FEATURES['DISABLE_START_DATES']: + course_target = reverse('info', args=[course.id]) + else: + course_target = reverse('about_course', args=[course.id]) + show_link = settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION') + %> + %if show_link: + + %endif You are registered for this course (${course.number}). + %if show_link: + + %endif %else: Register for ${course.number} +
    %endif %else: Register for ${course.number} diff --git a/lms/templates/profile.html b/lms/templates/profile.html index 1e3bde5969..8107bb1923 100644 --- a/lms/templates/profile.html +++ b/lms/templates/profile.html @@ -4,6 +4,7 @@ <%block name="headextra"> <%static:css group='course'/> + <%namespace name="profile_graphs" file="profile_graphs.js"/> <%block name="title">Profile - edX 6.002x @@ -71,7 +72,7 @@ $(function() { var new_email = $('#new_email_field').val(); var new_password = $('#new_email_password').val(); - postJSON('/change_email',{"new_email":new_email, + postJSON('/change_email',{"new_email":new_email, "password":new_password}, function(data){ if(data.success){ @@ -80,7 +81,7 @@ $(function() { $("#change_email_error").html(data.error); } }); - log_event("profile", {"type":"email_change_request", + log_event("profile", {"type":"email_change_request", "old_email":"${email}", "new_email":new_email}); return false; @@ -90,7 +91,7 @@ $(function() { var new_name = $('#new_name_field').val(); var rationale = $('#name_rationale_field').val(); - postJSON('/change_name',{"new_name":new_name, + postJSON('/change_name',{"new_name":new_name, "rationale":rationale}, function(data){ if(data.success){ @@ -99,7 +100,7 @@ $(function() { $("#change_name_error").html(data.error); } }); - log_event("profile", {"type":"name_change_request", + log_event("profile", {"type":"name_change_request", "new_name":new_name, "rationale":rationale}); return false; @@ -110,9 +111,9 @@ $(function() { -<%include file="navigation.html" args="active_page='profile'" /> +<%include file="course_navigation.html" args="active_page='profile'" /> -
    +
    @@ -124,10 +125,9 @@ $(function() {
      %for chapter in courseware_summary: - %if not chapter['chapter'] == "hidden": + %if not chapter['display_name'] == "hidden":
    1. -

      - ${ chapter['chapter'] }

      +

      ${ chapter['display_name'] }

        %for section in chapter['sections']: @@ -137,14 +137,14 @@ $(function() { total = section['section_total'].possible percentageString = "{0:.0%}".format( float(earned)/total) if earned > 0 and total > 0 else "" %> - -

        - ${ section['section'] } ${"({0:.3n}/{1:.3n}) {2}".format( float(earned), float(total), percentageString )}

        + +

        + ${ section['display_name'] } ${"({0:.3n}/{1:.3n}) {2}".format( float(earned), float(total), percentageString )}

        ${section['format']} %if 'due' in section and section['due']!="": due ${section['due']} %endif - + %if len(section['scores']) > 0:
          ${ "Problem Scores: " if section['graded'] else "Practice Scores: "} @@ -153,7 +153,7 @@ $(function() { %endfor
        %endif - + %endfor
      @@ -181,7 +181,7 @@ $(function() {
    2. - Forum name: ${username} + Forum name: ${username}
    3. @@ -215,7 +215,7 @@ $(function() {
      -

      To uphold the credibility of edX certificates, name changes must go through an approval process. A member of the course staff will review your request, and if approved, update your information. Please allow up to a week for your request to be processed. Thank you.

      +

      To uphold the credibility of edX certificates, name changes must go through an approval process. A member of the course staff will review your request, and if approved, update your information. Please allow up to a week for your request to be processed. Thank you.

      • @@ -234,7 +234,7 @@ $(function() {
    -

    Change e-mail

    +

    Change e-mail

    diff --git a/lms/templates/signup_modal.html b/lms/templates/signup_modal.html index aef90ab0f2..1510eb407b 100644 --- a/lms/templates/signup_modal.html +++ b/lms/templates/signup_modal.html @@ -19,6 +19,7 @@
    + % if has_extauth_info is UNDEFINED: @@ -27,6 +28,12 @@ + % else: +

    Welcome ${extauth_email}


    +

    Enter a public username:

    + + + % endif
    @@ -93,11 +100,13 @@
    + % if has_extauth_info is UNDEFINED: + % endif
    diff --git a/lms/templates/staff_problem_info.html b/lms/templates/staff_problem_info.html index f9fa999ae9..c9b92c51db 100644 --- a/lms/templates/staff_problem_info.html +++ b/lms/templates/staff_problem_info.html @@ -1,11 +1,11 @@ ${module_content} -
    -definition = ${definition | h} -metadata = ${metadata | h} -
    %if edit_link:
    Edit
    % endif +
    +definition = 
    ${definition | h}
    +metadata = ${metadata | h} +
    %if render_histogram:
    %endif diff --git a/lms/templates/staticbook.html b/lms/templates/staticbook.html index f5b184cc1c..eae70cdd84 100644 --- a/lms/templates/staticbook.html +++ b/lms/templates/staticbook.html @@ -67,7 +67,6 @@ $("#open_close_accordion a").click(function(){
    -

    Table of Contents

    close
    diff --git a/lms/templates/tracking_log.html b/lms/templates/tracking_log.html new file mode 100644 index 0000000000..66d375c2f3 --- /dev/null +++ b/lms/templates/tracking_log.html @@ -0,0 +1,14 @@ + +

    Tracking Log

    + +% for rec in records: + + + + + + + +% endfor +
    datetimeusernameipaddrsourcetype
    ${rec.time}${rec.username}${rec.ip}${rec.event_source}${rec.event_type}
    + \ No newline at end of file diff --git a/lms/templates/video.html b/lms/templates/video.html index 65ff44e8fa..93273ddb87 100644 --- a/lms/templates/video.html +++ b/lms/templates/video.html @@ -1,5 +1,5 @@ % if name is not UNDEFINED and name is not None: -

    ${name}

    +

    ${display_name}

    % endif
    diff --git a/lms/urls.py b/lms/urls.py index 1c4a065e2b..9dc317039e 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -14,6 +14,8 @@ urlpatterns = ('', url(r'^$', 'student.views.index', name="root"), # Main marketing page, or redirect to courseware url(r'^dashboard$', 'student.views.dashboard', name="dashboard"), + url(r'^admin_dashboard$', 'dashboard.views.dashboard'), + url(r'^change_email$', 'student.views.change_email_request'), url(r'^email_confirm/(?P[^/]*)$', 'student.views.confirm_email_change'), url(r'^change_name$', 'student.views.change_name_request'), @@ -160,12 +162,29 @@ if settings.DEBUG: ## Jasmine urlpatterns=urlpatterns + (url(r'^_jasmine/', include('django_jasmine.urls')),) +if settings.MITX_FEATURES.get('AUTH_USE_OPENID'): + urlpatterns += ( + url(r'^openid/login/$', 'django_openid_auth.views.login_begin', name='openid-login'), + url(r'^openid/complete/$', 'external_auth.views.edXauth_openid_login_complete', name='openid-complete'), + url(r'^openid/logo.gif$', 'django_openid_auth.views.logo', name='openid-logo'), + ) + +if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'): + urlpatterns += ( + url(r'^migrate/modules$', 'lms_migration.migrate.manage_modulestores'), + url(r'^migrate/reload/(?P[^/]+)$', 'lms_migration.migrate.manage_modulestores'), + ) + +if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'): + urlpatterns += ( + url(r'^event_logs$', 'track.views.view_tracking_log'), + ) + urlpatterns = patterns(*urlpatterns) if settings.DEBUG: urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) - #Custom error pages handler404 = 'static_template_view.views.render_404' handler500 = 'static_template_view.views.render_500' diff --git a/prod-requirements.txt b/prod-requirements.txt new file mode 100644 index 0000000000..98545552cb --- /dev/null +++ b/prod-requirements.txt @@ -0,0 +1,52 @@ +Django==1.3.1 +flup==1.0.3.dev-20110405 +lxml==2.3.4 +Mako==0.7.0 +Markdown==2.1.1 +markdown2==1.4.2 +python-memcached==1.48 +numpy==1.6.1 +Pygments==1.5 +boto==2.3.0 +django-storages==1.1.4 +django-masquerade==0.1.5 +fs==0.4.0 +django-jasmine==0.3.2 +path.py==2.2.2 +requests==0.12.1 +BeautifulSoup==3.2.1 +BeautifulSoup4==4.1.1 +newrelic==1.3.0.289 +ipython==0.12.1 +django-pipeline==1.2.12 +django-staticfiles==1.2.1 +glob2==0.3 +sympy==0.7.1 +pymongo==2.2.1 +rednose==0.3.3 +mock==0.8.0 +GitPython==0.3.2.RC1 +PyYAML==3.10 +feedparser==5.1.2 +MySQL-python==1.2.3 +matplotlib==1.1.0 +scipy==0.10.1 +akismet==0.2.0 +Coffin==0.3.6 +django-celery==2.2.7 +django-countries==1.0.5 +django-followit==0.0.3 +django-keyedcache==1.4-6 +django-kombu==0.9.2 +django-mako==0.1.5pre +django-recaptcha-works==0.3.4 +django-robots==0.8.1 +django-ses==0.4.1 +django-threaded-multihost==1.4-1 +html5lib==0.90 +Jinja2==2.6 +oauth2==1.5.211 +pystache==0.3.1 +python-openid==2.2.5 +South==0.7.5 +Unidecode==0.04.9 diff --git a/requirements.txt b/requirements.txt index 978b5e6f1a..33b2bfeb05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,19 +2,20 @@ django<1.4 pip numpy scipy -matplotlib markdown pygments lxml boto mako python-memcached +python-openid path.py django_debug_toolbar -e git://github.com/MITx/django-pipeline.git#egg=django-pipeline django-staticfiles>=1.2.1 fs -beautifulsoup +beautifulsoup +beautifulsoup4 feedparser requests sympy @@ -37,6 +38,7 @@ django-jasmine django-keyedcache django-mako django-masquerade +django-openid-auth django-robots django-ses django-storages diff --git a/utility-scripts/create_groups.py b/utility-scripts/create_groups.py new file mode 100644 index 0000000000..33c563127f --- /dev/null +++ b/utility-scripts/create_groups.py @@ -0,0 +1,35 @@ +#!/usr/bin/python +# +# File: create_groups.py +# +# Create all staff_* groups for classes in data directory. + +import os, sys, string, re + +sys.path.append(os.path.abspath('.')) +os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev' + +try: + from lms.envs.dev import * +except Exception as err: + print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory." + sys.exit(-1) + +from django.conf import settings +from django.contrib.auth.models import User, Group +from path import path + +data_dir = settings.DATA_DIR +print "data_dir = %s" % data_dir + +for course_dir in os.listdir(data_dir): + # print course_dir + if not os.path.isdir(path(data_dir) / course_dir): + continue + gname = 'staff_%s' % course_dir + if Group.objects.filter(name=gname): + print "group exists for %s" % gname + continue + g = Group(name=gname) + g.save() + print "created group %s" % gname diff --git a/utility-scripts/create_user.py b/utility-scripts/create_user.py new file mode 100644 index 0000000000..3ce9ce0ecf --- /dev/null +++ b/utility-scripts/create_user.py @@ -0,0 +1,149 @@ +#!/usr/bin/python +# +# File: create_user.py +# +# Create user. Prompt for groups and ExternalAuthMap + +import os, sys, string, re +import datetime +from getpass import getpass +import json +import readline + +sys.path.append(os.path.abspath('.')) +os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev' + +try: + from lms.envs.dev import * +except Exception as err: + print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory." + sys.exit(-1) + +from student.models import UserProfile, Registration +from external_auth.models import ExternalAuthMap +from django.contrib.auth.models import User, Group +from random import choice + +class MyCompleter(object): # Custom completer + + def __init__(self, options): + self.options = sorted(options) + + def complete(self, text, state): + if state == 0: # on first trigger, build possible matches + if text: # cache matches (entries that start with entered text) + self.matches = [s for s in self.options + if s and s.startswith(text)] + else: # no text entered, all matches possible + self.matches = self.options[:] + + # return match indexed by state + try: + return self.matches[state] + except IndexError: + return None + +def GenPasswd(length=8, chars=string.letters + string.digits): + return ''.join([choice(chars) for i in range(length)]) + +#----------------------------------------------------------------------------- +# main + +while True: + uname = raw_input('username: ') + if User.objects.filter(username=uname): + print "username %s already taken" % uname + else: + break + +make_eamap = False +if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y': + email = '%s@MIT.EDU' % uname + if not email.endswith('@MIT.EDU'): + print "Failed - email must be @MIT.EDU" + sys.exit(-1) + mit_domain = 'ssl:MIT' + if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain): + print "Failed - email %s already exists as external_id" % email + sys.exit(-1) + make_eamap = True + password = GenPasswd(12) + + # get name from kerberos + kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip() + name = raw_input('Full name: [%s] ' % kname).strip() + if name=='': + name = kname + print "name = %s" % name +else: + while True: + password = getpass() + password2 = getpass() + if password == password2: + break + print "Oops, passwords do not match, please retry" + + while True: + email = raw_input('email: ') + if User.objects.filter(email=email): + print "email %s already taken" % email + else: + break + + name = raw_input('Full name: ') + + +user = User(username=uname, email=email, is_active=True) +user.set_password(password) +try: + user.save() +except IntegrityError: + print "Oops, failed to create user %s, IntegrityError" % user + raise + +r = Registration() +r.register(user) + +up = UserProfile(user=user) +up.name = name +up.save() + +if make_eamap: + credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email) + eamap = ExternalAuthMap(external_id = email, + external_email = email, + external_domain = mit_domain, + external_name = name, + internal_password = password, + external_credentials = json.dumps(credentials), + ) + eamap.user = user + eamap.dtsignup = datetime.datetime.now() + eamap.save() + +print "User %s created successfully!" % user + +if not raw_input('Add user %s to any groups? [n] ' % user).lower()=='y': + sys.exit(0) + +print "Here are the groups available:" + +groups = [str(g.name) for g in Group.objects.all()] +print groups + +completer = MyCompleter(groups) +readline.set_completer(completer.complete) +readline.parse_and_bind('tab: complete') + +while True: + gname = raw_input("Add group (tab to autocomplete, empty line to end): ") + if not gname: + break + if not gname in groups: + print "Unknown group %s" % gname + continue + g = Group.objects.get(name=gname) + user.groups.add(g) + print "Added %s to group %s" % (user,g) + +print "Done!"