diff --git a/.gitignore b/.gitignore index 28b78aedbc..3653c832fa 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ Gemfile.lock .env/ lms/static/sass/*.css cms/static/sass/*.css +lms/lib/comment_client/python diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index 0305795e52..31be96ad7b 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -249,7 +249,7 @@ def load_preview_module(request, preview_id, descriptor, instance_state, shared_ module = descriptor.xmodule_constructor(system)(instance_state, shared_state) module.get_html = replace_static_urls( wrap_xmodule(module.get_html, module, "xmodule_display.html"), - module.metadata['data_dir'] + module.metadata['data_dir'], module ) save_preview_state(request, preview_id, descriptor.location.url(), module.get_instance_state(), module.get_shared_state()) diff --git a/cms/envs/common.py b/cms/envs/common.py index 6faecafec1..1767202141 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -90,6 +90,16 @@ TEMPLATE_CONTEXT_PROCESSORS = ( ################################# Jasmine ################################### JASMINE_TEST_DIRECTORY = PROJECT_ROOT + '/static/coffee' + +#################### CAPA External Code Evaluation ############################# +XQUEUE_INTERFACE = { + 'url': 'http://localhost:8888', + 'django_auth': {'username': 'local', + 'password': 'local'}, + 'basic_auth': None, +} + + ################################# Middleware ################################### # List of finder classes that know how to find static files in # various locations. diff --git a/cms/envs/test.py b/cms/envs/test.py index bce3c796cf..3823cd9dd9 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -55,6 +55,17 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': ENV_ROOT / "db" / "cms.db", + }, + + # The following are for testing purposes... + 'edX/toy/2012_Fall': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ENV_ROOT / "db" / "course1.db", + }, + + 'edx/full/6.002_Spring_2012': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ENV_ROOT / "db" / "course2.db", } } diff --git a/common/djangoapps/pipeline_mako/__init__.py b/common/djangoapps/pipeline_mako/__init__.py index 4703b53e52..1cdc287e2e 100644 --- a/common/djangoapps/pipeline_mako/__init__.py +++ b/common/djangoapps/pipeline_mako/__init__.py @@ -1,10 +1,9 @@ -from staticfiles.storage import staticfiles_storage - from mitxmako.shortcuts import render_to_string from pipeline.conf import settings from pipeline.packager import Packager from pipeline.utils import guess_type +from static_replace import try_staticfiles_lookup def compressed_css(package_name): @@ -25,9 +24,11 @@ def compressed_css(package_name): def render_css(package, path): template_name = package.template_name or "mako/css.html" context = package.extra_context + + url = try_staticfiles_lookup(path) context.update({ 'type': guess_type(path, 'text/css'), - 'url': staticfiles_storage.url(path) + 'url': url, }) return render_to_string(template_name, context) @@ -58,7 +59,7 @@ def render_js(package, path): context = package.extra_context context.update({ 'type': guess_type(path, 'text/javascript'), - 'url': staticfiles_storage.url(path) + 'url': try_staticfiles_lookup(path) }) return render_to_string(template_name, context) diff --git a/common/djangoapps/pipeline_mako/templates/static_content.html b/common/djangoapps/pipeline_mako/templates/static_content.html index 1737153260..c153da22fe 100644 --- a/common/djangoapps/pipeline_mako/templates/static_content.html +++ b/common/djangoapps/pipeline_mako/templates/static_content.html @@ -3,7 +3,13 @@ from staticfiles.storage import staticfiles_storage from pipeline_mako import compressed_css, compressed_js %> -<%def name='url(file)'>${staticfiles_storage.url(file)} +<%def name='url(file)'> +<% +try: + url = staticfiles_storage.url(file) +except: + url = file +%>${url} <%def name='css(group)'> % if settings.MITX_FEATURES['USE_DJANGO_PIPELINE']: diff --git a/common/djangoapps/static_replace.py b/common/djangoapps/static_replace.py index f9660e7f5e..ce3dc55031 100644 --- a/common/djangoapps/static_replace.py +++ b/common/djangoapps/static_replace.py @@ -1,6 +1,26 @@ -from staticfiles.storage import staticfiles_storage +import logging import re +from staticfiles.storage import staticfiles_storage +from staticfiles import finders +from django.conf import settings + +log = logging.getLogger(__name__) + +def try_staticfiles_lookup(path): + """ + Try to lookup a path in staticfiles_storage. If it fails, return + a dead link instead of raising an exception. + """ + try: + url = staticfiles_storage.url(path) + except Exception as err: + log.warning("staticfiles_storage couldn't find path {}: {}".format( + path, str(err))) + # Just return the original path; don't kill everything. + url = path + return url + def replace(static_url, prefix=None): if prefix is None: @@ -9,10 +29,19 @@ def replace(static_url, prefix=None): prefix = prefix + '/' quote = static_url.group('quote') - if staticfiles_storage.exists(static_url.group('rest')): + + servable = ( + # If in debug mode, we'll serve up anything that the finders can find + (settings.DEBUG and finders.find(static_url.group('rest'), True)) or + # Otherwise, we'll only serve up stuff that the storages can find + staticfiles_storage.exists(static_url.group('rest')) + ) + + if servable: return static_url.group(0) else: - url = staticfiles_storage.url(prefix + static_url.group('rest')) + # don't error if file can't be found + url = try_staticfiles_lookup(prefix + static_url.group('rest')) return "".join([quote, url, quote]) diff --git a/common/djangoapps/student/management/commands/create_random_users.py b/common/djangoapps/student/management/commands/create_random_users.py new file mode 100644 index 0000000000..c6cf452a43 --- /dev/null +++ b/common/djangoapps/student/management/commands/create_random_users.py @@ -0,0 +1,36 @@ +## +## A script to create some dummy users + +from django.core.management.base import BaseCommand +from django.conf import settings +from django.contrib.auth.models import User +from student.models import UserProfile, CourseEnrollment + +from student.views import _do_create_account, get_random_post_override + +def create(n, course_id): + """Create n users, enrolling them in course_id if it's not None""" + for i in range(n): + (user, user_profile, _) = _do_create_account(get_random_post_override()) + if course_id is not None: + CourseEnrollment.objects.create(user=user, course_id=course_id) + +class Command(BaseCommand): + help = """Create N new users, with random parameters. + +Usage: create_random_users.py N [course_id_to_enroll_in]. + +Examples: + create_random_users.py 1 + create_random_users.py 10 MITx/6.002x/2012_Fall + create_random_users.py 100 HarvardX/CS50x/2012 +""" + + def handle(self, *args, **options): + if len(args) < 1 or len(args) > 2: + print Command.help + return + + n = int(args[0]) + course_id = args[1] if len(args) == 2 else None + create(n, course_id) diff --git a/common/djangoapps/student/management/commands/sync_user_info.py b/common/djangoapps/student/management/commands/sync_user_info.py index 7d7dc4405d..04257e2a5d 100644 --- a/common/djangoapps/student/management/commands/sync_user_info.py +++ b/common/djangoapps/student/management/commands/sync_user_info.py @@ -4,7 +4,7 @@ from django.core.management.base import BaseCommand from django.contrib.auth.models import User -import comment_client +import comment_client as cc class Command(BaseCommand): @@ -15,8 +15,5 @@ service''' def handle(self, *args, **options): for user in User.objects.all().iterator(): - comment_client.update_user(user.id, { - 'id': user.id, - 'username': user.username, - 'email': user.email - }) + cc_user = cc.User.from_django_user(user) + cc_user.save() diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py index 5919d215c9..382b7ebcda 100644 --- a/common/djangoapps/student/models.py +++ b/common/djangoapps/student/models.py @@ -1,5 +1,30 @@ """ -WE'RE USING MIGRATIONS! +Models for Student Information + +Replication Notes + +In our live deployment, we intend to run in a scenario where there is a pool of +Portal servers that hold the canoncial user information and that user +information is replicated to slave Course server pools. Each Course has a set of +servers that serves only its content and has users that are relevant only to it. + +We replicate the following tables into the Course DBs where the user is +enrolled. Only the Portal servers should ever write to these models. +* UserProfile +* CourseEnrollment + +We do a partial replication of: +* User -- Askbot extends this and uses the extra fields, so we replicate only + the stuff that comes with basic django_auth and ignore the rest.) + +There are a couple different scenarios: + +1. There's an update of User or UserProfile -- replicate it to all Course DBs + that the user is enrolled in (found via CourseEnrollment). +2. There's a change in CourseEnrollment. We need to push copies of UserProfile, + CourseEnrollment, and the base fields in User + +Migration Notes If you make changes to this model, be sure to create an appropriate migration file and check it in at the same time as your model changes. To do that, @@ -10,10 +35,14 @@ file and check it in at the same time as your model changes. To do that, """ from datetime import datetime import json +import logging import uuid -from django.db import models +from django.conf import settings from django.contrib.auth.models import User +from django.db import models +from django.db.models.signals import post_delete, post_save +from django.dispatch import receiver from django_countries import CountryField from django.db.models.signals import post_save @@ -21,15 +50,35 @@ from django.dispatch import receiver from functools import partial -import comment_client +import comment_client as cc import logging +from xmodule.modulestore.django import modulestore #from cache_toolbox import cache_model, cache_relation +log = logging.getLogger(__name__) class UserProfile(models.Model): + """This is where we store all the user demographic fields. We have a + separate table for this rather than extending the built-in Django auth_user. + + Notes: + * Some fields are legacy ones from the first run of 6.002, from which + we imported many users. + * Fields like name and address are intentionally open ended, to account + for international variations. An unfortunate side-effect is that we + cannot efficiently sort on last names for instance. + + Replication: + * Only the Portal servers should ever modify this information. + * All fields are replicated into relevant Course databases + + Some of the fields are legacy ones that were captured during the initial + MITx fall prototype. + """ + class Meta: db_table = "auth_userprofile" @@ -216,17 +265,160 @@ def add_user_to_default_group(user, group): @receiver(post_save, sender=User) def update_user_information(sender, instance, created, **kwargs): - if created: - func = comment_client.create_user - else: - func = partial(comment_client.update_user, user_id=instance.id) try: - func(attributes={ - 'id': instance.id, - 'username': instance.username, - 'email': instance.email, - }) + cc_user = cc.User.from_django_user(instance) + cc_user.save() except Exception as e: log = logging.getLogger("mitx.discussion") log.error(unicode(e)) log.error("update user info to discussion failed for user with id: " + str(instance.id)) + +########################## REPLICATION SIGNALS ################################# +@receiver(post_save, sender=User) +def replicate_user_save(sender, **kwargs): + user_obj = kwargs['instance'] + if not should_replicate(user_obj): + return + for course_db_name in db_names_to_replicate_to(user_obj.id): + replicate_user(user_obj, course_db_name) + +@receiver(post_save, sender=CourseEnrollment) +def replicate_enrollment_save(sender, **kwargs): + """This is called when a Student enrolls in a course. It has to do the + following: + + 1. Make sure the User is copied into the Course DB. It may already exist + (someone deleting and re-adding a course). This has to happen first or + the foreign key constraint breaks. + 2. Replicate the CourseEnrollment. + 3. Replicate the UserProfile. + """ + if not is_portal(): + return + + enrollment_obj = kwargs['instance'] + log.debug("Replicating user because of new enrollment") + replicate_user(enrollment_obj.user, enrollment_obj.course_id) + + log.debug("Replicating enrollment because of new enrollment") + replicate_model(CourseEnrollment.save, enrollment_obj, enrollment_obj.user_id) + + log.debug("Replicating user profile because of new enrollment") + user_profile = UserProfile.objects.get(user_id=enrollment_obj.user_id) + replicate_model(UserProfile.save, user_profile, enrollment_obj.user_id) + +@receiver(post_delete, sender=CourseEnrollment) +def replicate_enrollment_delete(sender, **kwargs): + enrollment_obj = kwargs['instance'] + return replicate_model(CourseEnrollment.delete, enrollment_obj, enrollment_obj.user_id) + +@receiver(post_save, sender=UserProfile) +def replicate_userprofile_save(sender, **kwargs): + """We just updated the UserProfile (say an update to the name), so push that + change to all Course DBs that we're enrolled in.""" + user_profile_obj = kwargs['instance'] + return replicate_model(UserProfile.save, user_profile_obj, user_profile_obj.user_id) + + +######### Replication functions ######### +USER_FIELDS_TO_COPY = ["id", "username", "first_name", "last_name", "email", + "password", "is_staff", "is_active", "is_superuser", + "last_login", "date_joined"] + +def replicate_user(portal_user, course_db_name): + """Replicate a User to the correct Course DB. This is more complicated than + it should be because Askbot extends the auth_user table and adds its own + fields. So we need to only push changes to the standard fields and leave + the rest alone so that Askbot changes at the Course DB level don't get + overridden. + """ + try: + course_user = User.objects.using(course_db_name).get(id=portal_user.id) + log.debug("User {0} found in Course DB, replicating fields to {1}" + .format(course_user, course_db_name)) + except User.DoesNotExist: + log.debug("User {0} not found in Course DB, creating copy in {1}" + .format(portal_user, course_db_name)) + course_user = User() + + for field in USER_FIELDS_TO_COPY: + setattr(course_user, field, getattr(portal_user, field)) + + mark_handled(course_user) + course_user.save(using=course_db_name) + unmark(course_user) + +def replicate_model(model_method, instance, user_id): + """ + model_method is the model action that we want replicated. For instance, + UserProfile.save + """ + if not should_replicate(instance): + return + + course_db_names = db_names_to_replicate_to(user_id) + log.debug("Replicating {0} for user {1} to DBs: {2}" + .format(model_method, user_id, course_db_names)) + + mark_handled(instance) + for db_name in course_db_names: + model_method(instance, using=db_name) + unmark(instance) + +######### Replication Helpers ######### + +def is_valid_course_id(course_id): + """Right now, the only database that's not a course database is 'default'. + I had nicer checking in here originally -- it would scan the courses that + were in the system and only let you choose that. But it was annoying to run + tests with, since we don't have course data for some for our course test + databases. Hence the lazy version. + """ + return course_id != 'default' + +def is_portal(): + """Are we in the portal pool? Only Portal servers are allowed to replicate + their changes. For now, only Portal servers see multiple DBs, so we use + that to decide.""" + return len(settings.DATABASES) > 1 + +def db_names_to_replicate_to(user_id): + """Return a list of DB names that this user_id is enrolled in.""" + return [c.course_id + for c in CourseEnrollment.objects.filter(user_id=user_id) + if is_valid_course_id(c.course_id)] + +def marked_handled(instance): + """Have we marked this instance as being handled to avoid infinite loops + caused by saving models in post_save hooks for the same models?""" + return hasattr(instance, '_do_not_copy_to_course_db') and instance._do_not_copy_to_course_db + +def mark_handled(instance): + """You have to mark your instance with this function or else we'll go into + an infinite loop since we're putting listeners on Model saves/deletes and + the act of replication requires us to call the same model method. + + We create a _replicated attribute to differentiate the first save of this + model vs. the duplicate save we force on to the course database. Kind of + a hack -- suggestions welcome. + """ + instance._do_not_copy_to_course_db = True + +def unmark(instance): + """If we don't unmark a model after we do replication, then consecutive + save() calls won't be properly replicated.""" + instance._do_not_copy_to_course_db = False + +def should_replicate(instance): + """Should this instance be replicated? We need to be a Portal server and + the instance has to not have been marked_handled.""" + if marked_handled(instance): + # Basically, avoid an infinite loop. You should + log.debug("{0} should not be replicated because it's been marked" + .format(instance)) + return False + if not is_portal(): + log.debug("{0} should not be replicated because we're not a portal." + .format(instance)) + return False + return True diff --git a/common/djangoapps/student/tests.py b/common/djangoapps/student/tests.py index 501deb776c..b33678fbac 100644 --- a/common/djangoapps/student/tests.py +++ b/common/djangoapps/student/tests.py @@ -4,13 +4,195 @@ when you run "manage.py test". Replace this with more appropriate tests for your application. """ +import logging +from datetime import datetime from django.test import TestCase +from .models import User, UserProfile, CourseEnrollment, replicate_user, USER_FIELDS_TO_COPY + +COURSE_1 = 'edX/toy/2012_Fall' +COURSE_2 = 'edx/full/6.002_Spring_2012' + +log = logging.getLogger(__name__) + +class ReplicationTest(TestCase): + + multi_db = True + + def test_user_replication(self): + """Test basic user replication.""" + portal_user = User.objects.create_user('rusty', 'rusty@edx.org', 'fakepass') + portal_user.first_name='Rusty' + portal_user.last_name='Skids' + portal_user.is_staff=True + portal_user.is_active=True + portal_user.is_superuser=True + portal_user.last_login=datetime(2012, 1, 1) + portal_user.date_joined=datetime(2011, 1, 1) + # This is an Askbot field and will break if askbot is not included + + if hasattr(portal_user, 'seen_response_count'): + portal_user.seen_response_count = 10 + + portal_user.save(using='default') + + # We replicate this user to Course 1, then pull the same user and verify + # that the fields copied over properly. + replicate_user(portal_user, COURSE_1) + course_user = User.objects.using(COURSE_1).get(id=portal_user.id) + + # Make sure the fields we care about got copied over for this user. + for field in USER_FIELDS_TO_COPY: + self.assertEqual(getattr(portal_user, field), + getattr(course_user, field), + "{0} not copied from {1} to {2}".format( + field, portal_user, course_user + )) + + # This hasattr lameness is here because we don't want this test to be + # triggered when we're being run by CMS tests (Askbot doesn't exist + # there, so the test will fail). + # + # seen_response_count isn't a field we care about, so it shouldn't have + # been copied over. + if hasattr(portal_user, 'seen_response_count'): + portal_user.seen_response_count = 20 + replicate_user(portal_user, COURSE_1) + course_user = User.objects.using(COURSE_1).get(id=portal_user.id) + self.assertEqual(portal_user.seen_response_count, 20) + self.assertEqual(course_user.seen_response_count, 0) + + # Another replication should work for an email change however, since + # it's a field we care about. + portal_user.email = "clyde@edx.org" + replicate_user(portal_user, COURSE_1) + course_user = User.objects.using(COURSE_1).get(id=portal_user.id) + self.assertEqual(portal_user.email, course_user.email) + + # During this entire time, the user data should never have made it over + # to COURSE_2 + self.assertRaises(User.DoesNotExist, + User.objects.using(COURSE_2).get, + id=portal_user.id) + + + def test_enrollment_for_existing_user_info(self): + """Test the effect of Enrolling in a class if you've already got user + data to be copied over.""" + # Create our User + portal_user = User.objects.create_user('jack', 'jack@edx.org', 'fakepass') + portal_user.first_name = "Jack" + portal_user.save() + + # Set up our UserProfile info + portal_user_profile = UserProfile.objects.create( + user=portal_user, + name="Jack Foo", + level_of_education=None, + gender='m', + mailing_address=None, + goals="World domination", + ) + portal_user_profile.save() + + # Now let's see if creating a CourseEnrollment copies all the relevant + # data. + portal_enrollment = CourseEnrollment.objects.create(user=portal_user, + course_id=COURSE_1) + portal_enrollment.save() + + # Grab all the copies we expect + course_user = User.objects.using(COURSE_1).get(id=portal_user.id) + self.assertEquals(portal_user, course_user) + self.assertRaises(User.DoesNotExist, + User.objects.using(COURSE_2).get, + id=portal_user.id) + + course_enrollment = CourseEnrollment.objects.using(COURSE_1).get(id=portal_enrollment.id) + self.assertEquals(portal_enrollment, course_enrollment) + self.assertRaises(CourseEnrollment.DoesNotExist, + CourseEnrollment.objects.using(COURSE_2).get, + id=portal_enrollment.id) + + course_user_profile = UserProfile.objects.using(COURSE_1).get(id=portal_user_profile.id) + self.assertEquals(portal_user_profile, course_user_profile) + self.assertRaises(UserProfile.DoesNotExist, + UserProfile.objects.using(COURSE_2).get, + id=portal_user_profile.id) + + log.debug("Make sure our seen_response_count is not replicated.") + if hasattr(portal_user, 'seen_response_count'): + portal_user.seen_response_count = 200 + course_user = User.objects.using(COURSE_1).get(id=portal_user.id) + self.assertEqual(portal_user.seen_response_count, 200) + self.assertEqual(course_user.seen_response_count, 0) + portal_user.save() + + course_user = User.objects.using(COURSE_1).get(id=portal_user.id) + self.assertEqual(portal_user.seen_response_count, 200) + self.assertEqual(course_user.seen_response_count, 0) + + portal_user.email = 'jim@edx.org' + portal_user.save() + course_user = User.objects.using(COURSE_1).get(id=portal_user.id) + self.assertEqual(portal_user.email, 'jim@edx.org') + self.assertEqual(course_user.email, 'jim@edx.org') + + + + def test_enrollment_for_user_info_after_enrollment(self): + """Test the effect of modifying User data after you've enrolled.""" + # Create our User + portal_user = User.objects.create_user('patty', 'patty@edx.org', 'fakepass') + portal_user.first_name = "Patty" + portal_user.save() + + # Set up our UserProfile info + portal_user_profile = UserProfile.objects.create( + user=portal_user, + name="Patty Foo", + level_of_education=None, + gender='f', + mailing_address=None, + goals="World peace", + ) + portal_user_profile.save() + + # Now let's see if creating a CourseEnrollment copies all the relevant + # data when things are saved. + portal_enrollment = CourseEnrollment.objects.create(user=portal_user, + course_id=COURSE_1) + portal_enrollment.save() + + portal_user.last_name = "Bar" + portal_user.save() + portal_user_profile.gender = 'm' + portal_user_profile.save() + + # Grab all the copies we expect, and make sure it doesn't end up in + # places we don't expect. + course_user = User.objects.using(COURSE_1).get(id=portal_user.id) + self.assertEquals(portal_user, course_user) + self.assertRaises(User.DoesNotExist, + User.objects.using(COURSE_2).get, + id=portal_user.id) + + course_enrollment = CourseEnrollment.objects.using(COURSE_1).get(id=portal_enrollment.id) + self.assertEquals(portal_enrollment, course_enrollment) + self.assertRaises(CourseEnrollment.DoesNotExist, + CourseEnrollment.objects.using(COURSE_2).get, + id=portal_enrollment.id) + + course_user_profile = UserProfile.objects.using(COURSE_1).get(id=portal_user_profile.id) + self.assertEquals(portal_user_profile, course_user_profile) + self.assertRaises(UserProfile.DoesNotExist, + UserProfile.objects.using(COURSE_2).get, + id=portal_user_profile.id) + + + + + + -class SimpleTest(TestCase): - def test_basic_addition(self): - """ - Tests that 1 + 1 always equals 2. - """ - self.assertEqual(1 + 1, 2) diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py index 87490786c1..ea1770109b 100644 --- a/common/djangoapps/student/views.py +++ b/common/djangoapps/student/views.py @@ -1,13 +1,14 @@ import datetime +import feedparser +import itertools import json import logging import random import string import sys -import uuid -import feedparser +import time import urllib -import itertools +import uuid from django.conf import settings from django.contrib.auth import logout, authenticate, login @@ -26,17 +27,19 @@ from bs4 import BeautifulSoup from django.core.cache import cache from django_future.csrf import ensure_csrf_cookie -from student.models import Registration, UserProfile, PendingNameChange, PendingEmailChange, CourseEnrollment +from student.models import (Registration, UserProfile, + PendingNameChange, PendingEmailChange, + CourseEnrollment) from util.cache import cache_if_anonymous from xmodule.course_module import CourseDescriptor from xmodule.modulestore.exceptions import ItemNotFoundError from xmodule.modulestore.django import modulestore from xmodule.modulestore.exceptions import ItemNotFoundError -from models import Registration, UserProfile, PendingNameChange, PendingEmailChange, CourseEnrollment from datetime import date from collections import namedtuple -from courseware.courses import course_staff_group_name, has_staff_access_to_course, get_courses_by_university +from courseware.courses import (course_staff_group_name, has_staff_access_to_course, + get_courses_by_university) log = logging.getLogger("mitx.student") Article = namedtuple('Article', 'title url author image deck publication publish_date') @@ -47,7 +50,8 @@ def csrf_token(context): csrf_token = context.get('csrf_token', '') if csrf_token == 'NOTPROVIDED': return '' - return u'
' % (csrf_token) + return (u'
' % (csrf_token)) @ensure_csrf_cookie @@ -94,8 +98,9 @@ def main_index(extra_context = {}, user=None): context.update(extra_context) return render_to_response('index.html', context) -def course_from_id(id): - course_loc = CourseDescriptor.id_to_location(id) +def course_from_id(course_id): + """Return the CourseDescriptor corresponding to this course_id""" + course_loc = CourseDescriptor.id_to_location(course_id) return modulestore().get_item(course_loc) @@ -127,7 +132,7 @@ def dashboard(request): try: courses.append(course_from_id(enrollment.course_id)) except ItemNotFoundError: - log.error("User {0} enrolled in non-existant course {1}" + log.error("User {0} enrolled in non-existent course {1}" .format(user.username, enrollment.course_id)) message = "" @@ -158,18 +163,41 @@ def try_change_enrollment(request): @login_required def change_enrollment_view(request): + """Delegate to change_enrollment to actually do the work.""" return HttpResponse(json.dumps(change_enrollment(request))) +def enrollment_allowed(user, course): + """If the course has an enrollment period, check whether we are in it. + Also respects the DARK_LAUNCH setting""" + now = time.gmtime() + start = course.enrollment_start + end = course.enrollment_end + + if (start is None or now > start) and (end is None or now < end): + # in enrollment period. + return True + + if settings.MITX_FEATURES['DARK_LAUNCH']: + if has_staff_access_to_course(user, course): + # if dark launch, staff can enroll outside enrollment window + return True + return False + def change_enrollment(request): if request.method != "POST": raise Http404 - action = request.POST.get("enrollment_action", "") user = request.user + if not user.is_authenticated(): + raise Http404 + + action = request.POST.get("enrollment_action", "") + course_id = request.POST.get("course_id", None) if course_id == None: - return HttpResponse(json.dumps({'success': False, 'error': 'There was an error receiving the course id.'})) + return HttpResponse(json.dumps({'success': False, + 'error': 'There was an error receiving the course id.'})) if action == "enroll": # Make sure the course exists @@ -177,17 +205,25 @@ def change_enrollment(request): try: course = course_from_id(course_id) except ItemNotFoundError: - log.error("User {0} tried to enroll in non-existant course {1}" + log.warning("User {0} tried to enroll in non-existant course {1}" .format(user.username, enrollment.course_id)) return {'success': False, 'error': 'The course requested does not exist.'} if settings.MITX_FEATURES.get('ACCESS_REQUIRE_STAFF_FOR_COURSE'): - # require that user be in the staff_* group (or be an overall admin) to be able to enroll - # eg staff_6.002x or staff_6.00x - if not has_staff_access_to_course(user,course): + # require that user be in the staff_* group (or be an + # overall admin) to be able to enroll eg staff_6.002x or + # staff_6.00x + if not has_staff_access_to_course(user, course): staff_group = course_staff_group_name(course) - log.debug('user %s denied enrollment to %s ; not in %s' % (user,course.location.url(),staff_group)) - return {'success': False, 'error' : '%s membership required to access course.' % staff_group} + log.debug('user %s denied enrollment to %s ; not in %s' % ( + user, course.location.url(), staff_group)) + return {'success': False, + 'error' : '%s membership required to access course.' % staff_group} + + if not enrollment_allowed(user, course): + return {'success': False, + 'error': 'enrollment in {} not allowed at this time' + .format(course.display_name)} enrollment, created = CourseEnrollment.objects.get_or_create(user=user, course_id=course.id) return {'success': True} @@ -264,6 +300,7 @@ def logout_user(request): def change_setting(request): ''' JSON call to change a profile setting: Right now, location ''' + # TODO (vshnayder): location is no longer used up = UserProfile.objects.get(user=request.user) # request.user.profile_cache if 'location' in request.POST: up.location = request.POST['location'] @@ -272,6 +309,59 @@ def change_setting(request): return HttpResponse(json.dumps({'success': True, 'location': up.location, })) +def _do_create_account(post_vars): + """ + Given cleaned post variables, create the User and UserProfile objects, as well as the + registration for this user. + + Returns a tuple (User, UserProfile, Registration). + + Note: this function is also used for creating test users. + """ + user = User(username=post_vars['username'], + email=post_vars['email'], + is_active=False) + user.set_password(post_vars['password']) + registration = Registration() + # TODO: Rearrange so that if part of the process fails, the whole process fails. + # Right now, we can have e.g. no registration e-mail sent out and a zombie account + try: + user.save() + except IntegrityError: + js = {'success': False} + # Figure out the cause of the integrity error + if len(User.objects.filter(username=post_vars['username'])) > 0: + js['value'] = "An account with this username already exists." + js['field'] = 'username' + return HttpResponse(json.dumps(js)) + + if len(User.objects.filter(email=post_vars['email'])) > 0: + js['value'] = "An account with this e-mail already exists." + js['field'] = 'email' + return HttpResponse(json.dumps(js)) + + raise + + registration.register(user) + + profile = UserProfile(user=user) + profile.name = post_vars['name'] + profile.level_of_education = post_vars.get('level_of_education') + profile.gender = post_vars.get('gender') + profile.mailing_address = post_vars.get('mailing_address') + profile.goals = post_vars.get('goals') + + try: + profile.year_of_birth = int(post_vars['year_of_birth']) + except (ValueError, KeyError): + profile.year_of_birth = None # If they give us garbage, just ignore it instead + # of asking them to put an integer. + try: + profile.save() + except Exception: + log.exception("UserProfile creation failed for user {0}.".format(user.id)) + return (user, profile, registration) + @ensure_csrf_cookie def create_account(request, post_override=None): @@ -343,50 +433,14 @@ def create_account(request, post_override=None): js['field'] = 'username' return HttpResponse(json.dumps(js)) - u = User(username=post_vars['username'], - email=post_vars['email'], - is_active=False) - u.set_password(post_vars['password']) - r = Registration() - # TODO: Rearrange so that if part of the process fails, the whole process fails. - # Right now, we can have e.g. no registration e-mail sent out and a zombie account - try: - u.save() - except IntegrityError: - # Figure out the cause of the integrity error - if len(User.objects.filter(username=post_vars['username'])) > 0: - js['value'] = "An account with this username already exists." - js['field'] = 'username' - return HttpResponse(json.dumps(js)) - - if len(User.objects.filter(email=post_vars['email'])) > 0: - js['value'] = "An account with this e-mail already exists." - js['field'] = 'email' - return HttpResponse(json.dumps(js)) - - raise - - r.register(u) - - up = UserProfile(user=u) - up.name = post_vars['name'] - up.level_of_education = post_vars.get('level_of_education') - up.gender = post_vars.get('gender') - up.mailing_address = post_vars.get('mailing_address') - up.goals = post_vars.get('goals') - - try: - up.year_of_birth = int(post_vars['year_of_birth']) - except (ValueError, KeyError): - up.year_of_birth = None # If they give us garbage, just ignore it instead - # of asking them to put an integer. - try: - up.save() - except Exception: - log.exception("UserProfile creation failed for user {0}.".format(u.id)) + # Ok, looks like everything is legit. Create the account. + ret = _do_create_account(post_vars) + if isinstance(ret,HttpResponse): # if there was an error then return that + return ret + (user, profile, registration) = ret d = {'name': post_vars['name'], - 'key': r.activation_key, + 'key': registration.activation_key, } # composes activation email @@ -398,10 +452,11 @@ def create_account(request, post_override=None): try: if settings.MITX_FEATURES.get('REROUTE_ACTIVATION_EMAIL'): dest_addr = settings.MITX_FEATURES['REROUTE_ACTIVATION_EMAIL'] - message = "Activation for %s (%s): %s\n" % (u, u.email, up.name) + '-' * 80 + '\n\n' + message + message = ("Activation for %s (%s): %s\n" % (user, user.email, profile.name) + + '-' * 80 + '\n\n' + message) send_mail(subject, message, settings.DEFAULT_FROM_EMAIL, [dest_addr], fail_silently=False) elif not settings.GENERATE_RANDOM_USER_CREDENTIALS: - res = u.email_user(subject, message, settings.DEFAULT_FROM_EMAIL) + res = user.email_user(subject, message, settings.DEFAULT_FROM_EMAIL) except: log.exception(sys.exc_info()) js['value'] = 'Could not send activation e-mail.' @@ -431,24 +486,30 @@ def create_account(request, post_override=None): return HttpResponse(json.dumps(js), mimetype="application/json") -def create_random_account(create_account_function): - +def get_random_post_override(): + """ + Return a dictionary suitable for passing to post_vars of _do_create_account or post_override + of create_account, with random user info. + """ def id_generator(size=6, chars=string.ascii_uppercase + string.ascii_lowercase + string.digits): return ''.join(random.choice(chars) for x in range(size)) - def inner_create_random_account(request): - post_override = {'username': "random_" + id_generator(), - 'email': id_generator(size=10, chars=string.ascii_lowercase) + "_dummy_test@mitx.mit.edu", - 'password': id_generator(), - 'location': id_generator(size=5, chars=string.ascii_uppercase), - 'name': id_generator(size=5, chars=string.ascii_lowercase) + " " + id_generator(size=7, chars=string.ascii_lowercase), - 'honor_code': u'true', - 'terms_of_service': u'true', } + return {'username': "random_" + id_generator(), + 'email': id_generator(size=10, chars=string.ascii_lowercase) + "_dummy_test@mitx.mit.edu", + 'password': id_generator(), + 'name': (id_generator(size=5, chars=string.ascii_lowercase) + " " + + id_generator(size=7, chars=string.ascii_lowercase)), + 'honor_code': u'true', + 'terms_of_service': u'true', } - return create_account_function(request, post_override=post_override) + +def create_random_account(create_account_function): + def inner_create_random_account(request): + return create_account_function(request, post_override=get_random_post_override()) return inner_create_random_account +# TODO (vshnayder): do we need GENERATE_RANDOM_USER_CREDENTIALS for anything? if settings.GENERATE_RANDOM_USER_CREDENTIALS: create_account = create_random_account(create_account) @@ -514,7 +575,7 @@ def reactivation_email(request): subject = ''.join(subject.splitlines()) message = render_to_string('reactivation_email.txt', d) - res = u.email_user(subject, message, settings.DEFAULT_FROM_EMAIL) + res = user.email_user(subject, message, settings.DEFAULT_FROM_EMAIL) return HttpResponse(json.dumps({'success': True})) diff --git a/common/djangoapps/xmodule_modifiers.py b/common/djangoapps/xmodule_modifiers.py index 380388b545..0aeaa59d69 100644 --- a/common/djangoapps/xmodule_modifiers.py +++ b/common/djangoapps/xmodule_modifiers.py @@ -34,7 +34,7 @@ def wrap_xmodule(get_html, module, template): return _get_html -def replace_static_urls(get_html, prefix): +def replace_static_urls(get_html, prefix, module): """ Updates the supplied module with a new get_html function that wraps the old get_html function and substitutes urls of the form /static/... @@ -69,14 +69,14 @@ def grade_histogram(module_id): return grades -def add_histogram(get_html, module): +def add_histogram(get_html, module, user): """ Updates the supplied module with a new get_html function that wraps the output of the old get_html function with additional information for admin users only, including a histogram of student answers and the definition of the xmodule - Does nothing if module is a SequenceModule + Does nothing if module is a SequenceModule or a VerticalModule. """ @wraps(get_html) def _get_html(): @@ -97,14 +97,20 @@ def add_histogram(get_html, module): # doesn't like symlinks) filepath = filename data_dir = osfs.root_path.rsplit('/')[-1] - edit_link = "https://github.com/MITx/%s/tree/master/%s" % (data_dir,filepath) + giturl = module.metadata.get('giturl','https://github.com/MITx') + edit_link = "%s/%s/tree/master/%s" % (giturl,data_dir,filepath) else: edit_link = False staff_context = {'definition': module.definition.get('data'), 'metadata': json.dumps(module.metadata, indent=4), - 'element_id': module.location.html_id(), + 'location': module.location, + 'xqa_key': module.metadata.get('xqa_key',''), + 'category': str(module.__class__.__name__), + 'element_id': module.location.html_id().replace('-','_'), 'edit_link': edit_link, + 'user': user, + 'xqa_server' : settings.MITX_FEATURES.get('USE_XQA_SERVER','http://xqa:server@content-qa.mitx.mit.edu/xqa'), 'histogram': json.dumps(histogram), 'render_histogram': render_histogram, 'module_content': get_html()} diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 92823667e7..82eb330174 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -39,9 +39,9 @@ import responsetypes # dict of tagname, Response Class -- this should come from auto-registering response_tag_dict = dict([(x.response_tag, x) for x in responsetypes.__all__]) -entry_types = ['textline', 'schematic', 'textbox', 'imageinput', 'optioninput', 'choicegroup', 'radiogroup', 'checkboxgroup', 'filesubmission'] +entry_types = ['textline', 'schematic', 'textbox', 'imageinput', 'optioninput', 'choicegroup', 'radiogroup', 'checkboxgroup', 'filesubmission', 'javascriptinput'] solution_types = ['solution'] # extra things displayed after "show answers" is pressed -response_properties = ["responseparam", "answer"] # these get captured as student responses +response_properties = ["codeparam", "responseparam", "answer"] # these get captured as student responses # special problem tags which should be turned into innocuous HTML html_transforms = {'problem': {'tag': 'div'}, @@ -57,7 +57,7 @@ global_context = {'random': random, 'eia': eia} # These should be removed from HTML output, including all subelements -html_problem_semantics = ["responseparam", "answer", "script", "hintgroup"] +html_problem_semantics = ["codeparam", "responseparam", "answer", "script", "hintgroup"] log = logging.getLogger('mitx.' + __name__) @@ -154,21 +154,10 @@ class LoncapaProblem(object): def get_max_score(self): ''' Return maximum score for this problem. - We do this by counting the number of answers available for each question - in the problem. If the Response for a question has a get_max_score() method - then we call that and add its return value to the count. That can be - used to give complex problems (eg programming questions) multiple points. ''' maxscore = 0 for response, responder in self.responders.iteritems(): - if hasattr(responder, 'get_max_score'): - try: - maxscore += responder.get_max_score() - except Exception: - log.debug('responder %s failed to properly return from get_max_score()' % responder) # FIXME - raise - else: - maxscore += len(self.responder_answers[response]) + maxscore += responder.get_max_score() return maxscore def get_score(self): @@ -229,14 +218,14 @@ class LoncapaProblem(object): Calls the Response for each question in this problem, to do the actual grading. ''' - + self.student_answers = convert_files_to_filenames(answers) - + oldcmap = self.correct_map # old CorrectMap newcmap = CorrectMap() # start new with empty CorrectMap # log.debug('Responders: %s' % self.responders) for responder in self.responders.values(): # Call each responsetype instance to do actual grading - if 'filesubmission' in responder.allowed_inputfields: # File objects are passed only if responsetype + if 'filesubmission' in responder.allowed_inputfields: # File objects are passed only if responsetype # explicitly allows for file submissions results = responder.evaluate_answers(answers, oldcmap) else: @@ -295,9 +284,9 @@ class LoncapaProblem(object): try: ifp = self.system.filestore.open(file) # open using ModuleSystem OSFS filestore except Exception as err: - log.error('Error %s in problem xml include: %s' % ( + log.warning('Error %s in problem xml include: %s' % ( err, etree.tostring(inc, pretty_print=True))) - log.error('Cannot find file %s in %s' % ( + log.warning('Cannot find file %s in %s' % ( file, self.system.filestore)) # if debugging, don't fail - just log error # TODO (vshnayder): need real error handling, display to users @@ -306,11 +295,11 @@ class LoncapaProblem(object): else: continue try: - incxml = etree.XML(ifp.read()) # read in and convert to XML + incxml = etree.XML(ifp.read()) # read in and convert to XML except Exception as err: - log.error('Error %s in problem xml include: %s' % ( + log.warning('Error %s in problem xml include: %s' % ( err, etree.tostring(inc, pretty_print=True))) - log.error('Cannot parse XML in %s' % (file)) + log.warning('Cannot parse XML in %s' % (file)) # if debugging, don't fail - just log error # TODO (vshnayder): same as above if not self.system.get('DEBUG'): @@ -393,9 +382,10 @@ class LoncapaProblem(object): context['script_code'] += code # store code source in context try: exec code in context, context # use "context" for global context; thus defs in code are global within code - except Exception: + except Exception as err: log.exception("Error while execing script code: " + code) - raise responsetypes.LoncapaProblemError("Error while executing script code") + msg = "Error while executing script code: %s" % str(err).replace('<','<') + raise responsetypes.LoncapaProblemError(msg) finally: sys.path = original_path diff --git a/common/lib/capa/capa/inputtypes.py b/common/lib/capa/capa/inputtypes.py index 5092e5c378..8c513e7aec 100644 --- a/common/lib/capa/capa/inputtypes.py +++ b/common/lib/capa/capa/inputtypes.py @@ -11,6 +11,7 @@ Module containing the problem elements which render into input objects - choicegroup - radiogroup - checkboxgroup +- javascriptinput - imageinput (for clickable image) - optioninput (for option list) - filesubmission (upload a file) @@ -246,6 +247,34 @@ def checkboxgroup(element, value, status, render_template, msg=''): html = render_template("choicegroup.html", context) return etree.XML(html) +@register_render_function +def javascriptinput(element, value, status, render_template, msg='null'): + ''' + Hidden field for javascript to communicate via; also loads the required + scripts for rendering the problem and passes data to the problem. + ''' + eid = element.get('id') + params = element.get('params') + problem_state = element.get('problem_state') + display_class = element.get('display_class') + display_file = element.get('display_file') + + # Need to provide a value that JSON can parse if there is no + # student-supplied value yet. + if value == "": + value = 'null' + + escapedict = {'"': '"'} + value = saxutils.escape(value, escapedict) + msg = saxutils.escape(msg, escapedict) + context = {'id': eid, 'params': params, 'display_file': display_file, + 'display_class': display_class, 'problem_state': problem_state, + 'value': value, 'evaluation': msg, + } + html = render_template("javascriptinput.html", context) + return etree.XML(html) + + @register_render_function def textline(element, value, status, render_template, msg=""): @@ -307,9 +336,19 @@ def filesubmission(element, value, status, render_template, msg=''): Upload a single file (e.g. for programming assignments) ''' eid = element.get('id') - context = { 'id': eid, 'state': status, 'msg': msg, 'value': value, } + + # Check if problem has been queued + queue_len = 0 + if status == 'incomplete': # Flag indicating that the problem has been queued, 'msg' is length of queue + status = 'queued' + queue_len = msg + msg = 'Submitted to grader. (Queue length: %s)' % queue_len + + context = { 'id': eid, 'state': status, 'msg': msg, 'value': value, + 'queue_len': queue_len + } html = render_template("filesubmission.html", context) - return etree.XML(html) + return etree.XML(html) #----------------------------------------------------------------------------- @@ -330,9 +369,16 @@ def textbox(element, value, status, render_template, msg=''): if not value: value = element.text # if no student input yet, then use the default input given by the problem + # Check if problem has been queued + queue_len = 0 + if status == 'incomplete': # Flag indicating that the problem has been queued, 'msg' is length of queue + status = 'queued' + queue_len = msg + msg = 'Submitted to grader. (Queue length: %s)' % queue_len + # For CodeMirror - mode = element.get('mode') or 'python' # mode, eg "python" or "xml" - linenumbers = element.get('linenumbers','true') # for CodeMirror + mode = element.get('mode','python') + linenumbers = element.get('linenumbers','true') tabsize = element.get('tabsize','4') tabsize = int(tabsize) @@ -340,6 +386,7 @@ def textbox(element, value, status, render_template, msg=''): 'mode': mode, 'linenumbers': linenumbers, 'rows': rows, 'cols': cols, 'hidden': hidden, 'tabsize': tabsize, + 'queue_len': queue_len, } html = render_template("textbox.html", context) try: diff --git a/common/lib/capa/capa/javascript_problem_generator.js b/common/lib/capa/capa/javascript_problem_generator.js new file mode 100644 index 0000000000..8c8d39b19f --- /dev/null +++ b/common/lib/capa/capa/javascript_problem_generator.js @@ -0,0 +1,30 @@ +require('coffee-script'); +var importAll = function (modulePath) { + module = require(modulePath); + for(key in module){ + global[key] = module[key]; + } +} + +importAll("mersenne-twister-min"); +importAll("xproblem"); + +generatorModulePath = process.argv[2]; +dependencies = JSON.parse(process.argv[3]); +seed = process.argv[4]; +params = JSON.parse(process.argv[5]); + +if(seed==null){ + seed = 4; +}else{ + seed = parseInt(seed); +} + +for(var i = 0; i < dependencies.length; i++){ + importAll(dependencies[i]); +} + +generatorModule = require(generatorModulePath); +generatorClass = generatorModule.generatorClass; +generator = new generatorClass(seed, params); +console.log(JSON.stringify(generator.generate())); diff --git a/common/lib/capa/capa/javascript_problem_grader.js b/common/lib/capa/capa/javascript_problem_grader.js new file mode 100644 index 0000000000..4f42466167 --- /dev/null +++ b/common/lib/capa/capa/javascript_problem_grader.js @@ -0,0 +1,26 @@ +require('coffee-script'); +var importAll = function (modulePath) { + module = require(modulePath); + for(key in module){ + global[key] = module[key]; + } +} + +importAll("xproblem"); + +graderModulePath = process.argv[2]; +dependencies = JSON.parse(process.argv[3]); +submission = JSON.parse(process.argv[4]); +problemState = JSON.parse(process.argv[5]); +params = JSON.parse(process.argv[6]); + +for(var i = 0; i < dependencies.length; i++){ + importAll(dependencies[i]); +} + +graderModule = require(graderModulePath); +graderClass = graderModule.graderClass; +grader = new graderClass(submission, problemState, params); +console.log(JSON.stringify(grader.grade())); +console.log(JSON.stringify(grader.evaluation)); +console.log(JSON.stringify(grader.solution)); diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 66212f1e87..d327b80c14 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -17,7 +17,11 @@ import random import re import requests import traceback +import hashlib import abc +import os +import subprocess +import xml.sax.saxutils as saxutils # specific library imports from calc import evaluator, UndefinedVariable @@ -71,7 +75,6 @@ class LoncapaResponse(object): In addition, these methods are optional: - - get_max_score : if defined, this is called to obtain the maximum score possible for this question - setup_response : find and note the answer input field IDs for the response; called by __init__ - check_hint_condition : check to see if the student's answers satisfy a particular condition for a hint to be displayed - render_html : render this Response as HTML (must return XHTML compliant string) @@ -130,6 +133,11 @@ class LoncapaResponse(object): if self.max_inputfields == 1: self.answer_id = self.answer_ids[0] # for convenience + self.maxpoints = dict() + for inputfield in self.inputfields: + maxpoints = inputfield.get('points','1') # By default, each answerfield is worth 1 point + self.maxpoints.update({inputfield.get('id'): int(maxpoints)}) + self.default_answer_map = {} # dict for default answer map (provided in input elements) for entry in self.inputfields: answer = entry.get('correct_answer') @@ -139,6 +147,12 @@ class LoncapaResponse(object): if hasattr(self, 'setup_response'): self.setup_response() + def get_max_score(self): + ''' + Return the total maximum points of all answer fields under this Response + ''' + return sum(self.maxpoints.values()) + def render_html(self, renderer): ''' Return XHTML Element tree representation of this Response. @@ -272,9 +286,190 @@ class LoncapaResponse(object): #----------------------------------------------------------------------------- + +class JavascriptResponse(LoncapaResponse): + ''' + This response type is used when the student's answer is graded via + Javascript using Node.js. + ''' + + response_tag = 'javascriptresponse' + max_inputfields = 1 + allowed_inputfields = ['javascriptinput'] + + def setup_response(self): + + # Sets up generator, grader, display, and their dependencies. + self.parse_xml() + + self.compile_display_javascript() + + self.params = self.extract_params() + + if self.generator: + self.problem_state = self.generate_problem_state() + else: + self.problem_state = None + + self.solution = None + + self.prepare_inputfield() + + def compile_display_javascript(self): + + latestTimestamp = 0 + basepath = self.system.filestore.root_path + '/js/' + for filename in (self.display_dependencies + [self.display]): + filepath = basepath + filename + timestamp = os.stat(filepath).st_mtime + if timestamp > latestTimestamp: + latestTimestamp = timestamp + + h = hashlib.md5() + h.update(self.answer_id + str(self.display_dependencies)) + compiled_filename = 'compiled/' + h.hexdigest() + '.js' + compiled_filepath = basepath + compiled_filename + + if not os.path.exists(compiled_filepath) or os.stat(compiled_filepath).st_mtime < latestTimestamp: + outfile = open(compiled_filepath, 'w') + for filename in (self.display_dependencies + [self.display]): + filepath = basepath + filename + infile = open(filepath, 'r') + outfile.write(infile.read()) + outfile.write(';\n') + infile.close() + outfile.close() + + self.display_filename = compiled_filename + + def parse_xml(self): + self.generator_xml = self.xml.xpath('//*[@id=$id]//generator', + id=self.xml.get('id'))[0] + + self.grader_xml = self.xml.xpath('//*[@id=$id]//grader', + id=self.xml.get('id'))[0] + + self.display_xml = self.xml.xpath('//*[@id=$id]//display', + id=self.xml.get('id'))[0] + + self.xml.remove(self.generator_xml) + self.xml.remove(self.grader_xml) + self.xml.remove(self.display_xml) + + self.generator = self.generator_xml.get("src") + self.grader = self.grader_xml.get("src") + self.display = self.display_xml.get("src") + + if self.generator_xml.get("dependencies"): + self.generator_dependencies = self.generator_xml.get("dependencies").split() + else: + self.generator_dependencies = [] + + if self.grader_xml.get("dependencies"): + self.grader_dependencies = self.grader_xml.get("dependencies").split() + else: + self.grader_dependencies = [] + + if self.display_xml.get("dependencies"): + self.display_dependencies = self.display_xml.get("dependencies").split() + else: + self.display_dependencies = [] + + self.display_class = self.display_xml.get("class") + + def get_node_env(self): + + js_dir = os.path.join(self.system.filestore.root_path, 'js') + tmp_env = os.environ.copy() + node_path = self.system.node_path + ":" + os.path.normpath(js_dir) + tmp_env["NODE_PATH"] = node_path + return tmp_env + + + def generate_problem_state(self): + + generator_file = os.path.dirname(os.path.normpath(__file__)) + '/javascript_problem_generator.js' + output = subprocess.check_output(["node", + generator_file, + self.generator, + json.dumps(self.generator_dependencies), + json.dumps(str(self.system.seed)), + json.dumps(self.params) + ], + env=self.get_node_env()).strip() + + return json.loads(output) + + def extract_params(self): + + params = {} + + for param in self.xml.xpath('//*[@id=$id]//responseparam', + id=self.xml.get('id')): + + params[param.get("name")] = json.loads(param.get("value")) + + return params + + def prepare_inputfield(self): + + for inputfield in self.xml.xpath('//*[@id=$id]//javascriptinput', + id=self.xml.get('id')): + + escapedict = {'"': '"'} + + encoded_params = json.dumps(self.params) + encoded_params = saxutils.escape(encoded_params, escapedict) + inputfield.set("params", encoded_params) + + encoded_problem_state = json.dumps(self.problem_state) + encoded_problem_state = saxutils.escape(encoded_problem_state, + escapedict) + inputfield.set("problem_state", encoded_problem_state) + + inputfield.set("display_file", self.display_filename) + inputfield.set("display_class", self.display_class) + + def get_score(self, student_answers): + json_submission = student_answers[self.answer_id] + (all_correct, evaluation, solution) = self.run_grader(json_submission) + self.solution = solution + correctness = 'correct' if all_correct else 'incorrect' + return CorrectMap(self.answer_id, correctness, msg=evaluation) + + def run_grader(self, submission): + if submission is None or submission == '': + submission = json.dumps(None) + + grader_file = os.path.dirname(os.path.normpath(__file__)) + '/javascript_problem_grader.js' + outputs = subprocess.check_output(["node", + grader_file, + self.grader, + json.dumps(self.grader_dependencies), + submission, + json.dumps(self.problem_state), + json.dumps(self.params) + ], + env=self.get_node_env()).split('\n') + + all_correct = json.loads(outputs[0].strip()) + evaluation = outputs[1].strip() + solution = outputs[2].strip() + return (all_correct, evaluation, solution) + + def get_answers(self): + if self.solution is None: + (_, _, self.solution) = self.run_grader(None) + + return {self.answer_id: self.solution} + + + +#----------------------------------------------------------------------------- + class ChoiceResponse(LoncapaResponse): ''' - This Response type is used when the student chooses from a discrete set of + This response type is used when the student chooses from a discrete set of choices. Currently, to be marked correct, all "correct" choices must be supplied by the student, and no extraneous choices may be included. @@ -313,6 +508,11 @@ class ChoiceResponse(LoncapaResponse): In the above example, radiogroup can be replaced with checkboxgroup to allow the student to select more than one choice. + TODO: In order for the inputtypes to render properly, this response type + must run setup_response prior to the input type rendering. Specifically, the + choices must be given names. This behavior seems like a leaky abstraction, + and it'd be nice to change this at some point. + ''' response_tag = 'choiceresponse' @@ -668,7 +868,10 @@ def sympy_check2(): # if there is only one box, and it's empty, then don't evaluate if len(idset) == 1 and not submission[0]: - return CorrectMap(idset[0], 'incorrect', msg='No answer entered!') + # default to no error message on empty answer (to be consistent with other responsetypes) + # but allow author to still have the old behavior by setting empty_answer_err attribute + msg = 'No answer entered!' if self.xml.get('empty_answer_err') else '' + return CorrectMap(idset[0], 'incorrect', msg=msg) correct = ['unknown'] * len(idset) messages = [''] * len(idset) @@ -824,41 +1027,70 @@ class CodeResponse(LoncapaResponse): self.url = xml.get('url', None) # XML can override external resource (grader/queue) URL self.queue_name = xml.get('queuename', self.system.xqueue['default_queuename']) - self._parse_externalresponse_xml() + # VS[compat]: + # Check if XML uses the ExternalResponse format or the generic CodeResponse format + codeparam = self.xml.find('codeparam') + if codeparam is None: + self._parse_externalresponse_xml() + else: + self._parse_coderesponse_xml(codeparam) + + def _parse_coderesponse_xml(self,codeparam): + ''' + Parse the new CodeResponse XML format. When successful, sets: + self.initial_display + self.answer (an answer to display to the student in the LMS) + self.payload + ''' + # Note that CodeResponse is agnostic to the specific contents of grader_payload + grader_payload = codeparam.find('grader_payload') + grader_payload = grader_payload.text if grader_payload is not None else '' + self.payload = {'grader_payload': grader_payload} + + answer_display = codeparam.find('answer_display') + if answer_display is not None: + self.answer = answer_display.text + else: + self.answer = 'No answer provided.' + + initial_display = codeparam.find('initial_display') + if initial_display is not None: + self.initial_display = initial_display.text + else: + self.initial_display = '' def _parse_externalresponse_xml(self): ''' VS[compat]: Suppport for old ExternalResponse XML format. When successful, sets: - self.code - self.tests - self.answer self.initial_display + self.answer (an answer to display to the student in the LMS) + self.payload ''' answer = self.xml.find('answer') if answer is not None: answer_src = answer.get('src') if answer_src is not None: - self.code = self.system.filesystem.open('src/' + answer_src).read() + code = self.system.filesystem.open('src/' + answer_src).read() else: - self.code = answer.text + code = answer.text else: # no stanza; get code from +{# + +#} diff --git a/lms/askbot/skins/mitx/templates/navigation.jinja.html b/lms/askbot/skins/mitx/templates/navigation.jinja.html index 59c7148184..686ae3a724 100644 --- a/lms/askbot/skins/mitx/templates/navigation.jinja.html +++ b/lms/askbot/skins/mitx/templates/navigation.jinja.html @@ -1,25 +1,25 @@
+
diff --git a/lms/templates/discussion/thread.html b/lms/templates/discussion/thread.html deleted file mode 100644 index beee18a4ca..0000000000 --- a/lms/templates/discussion/thread.html +++ /dev/null @@ -1,107 +0,0 @@ -<%! from django.core.urlresolvers import reverse %> -<%! from datehelper import time_ago_in_words %> -<%! from dateutil.parser import parse %> -<%! import urllib %> - -<%def name="render_thread(course_id, thread, show_comments=False)"> -
- ${render_content(thread, "thread", show_comments=show_comments)} - % if show_comments: - ${render_comments(thread['children'])} - % endif -
- - -<%def name="render_comments(comments)"> -
- % for comment in comments: -
- ${render_content(comment, "comment")} -
- ${render_comments(comment['children'])} -
-
- % endfor -
- - -<%def name="render_content(content, type, **kwargs)"> -
-
- ${render_vote(content)} -
- ${render_title(content, type, **kwargs)} -
-
${content['body'] | h}
- - ${render_tags(content, type, **kwargs)} - ${render_bottom_bar(content, type, **kwargs)} -
-
-
-
- - -<%def name="render_title(content, type, **kwargs)"> - <% - if type == "thread": - if kwargs.get('show_comments', False): - url_for_thread = "" - else: - url_for_thread = reverse('django_comment_client.forum.views.single_thread', args=[course_id, content['commentable_id'], content['id']]) - %> - % if type == "thread": - ${content['title'] | h} - % endif - - -<%def name="render_tags(content, type, **kwargs)"> - <% - def url_for_tags(tags): - return reverse('django_comment_client.forum.views.search', args=[course_id]) + '?' + urllib.urlencode({'tags': ",".join(tags)}) - %> - % if type == "thread": -
- % for tag in content['tags']: - ${tag | h} - % endfor -
- - % endif - - -<%def name="render_bottom_bar(content, type, **kwargs)"> -
- ${render_info(content)} - ${render_link("discussion-link discussion-reply", "Reply")} - ${render_link("discussion-link discussion-edit", "Edit")} -
- - -<%def name="render_info(content)"> -
- ${time_ago_in_words(parse(content['updated_at']))} ago by - % if content['anonymous']: - anonymous - % else: - user No.${content['user_id']} - % endif -
-
- % if content.get('comments_count', -1) >= 0: - ${content['comments_count']} comment(s) - % endif -
- - -<%def name="render_link(cls, html)"> - ${html} - - -<%def name="render_vote(content)"> -
- ${render_link("discussion-vote discussion-vote-up", "˄")} - ${content['votes']['point']} - ${render_link("discussion-vote discussion-vote-down", "˅")} -
- diff --git a/lms/templates/discussion/user_profile.html b/lms/templates/discussion/user_profile.html new file mode 100644 index 0000000000..3825879c3b --- /dev/null +++ b/lms/templates/discussion/user_profile.html @@ -0,0 +1,40 @@ +<%! from django.template.defaultfilters import escapejs %> +<%namespace name="renderer" file="_thread.html"/> + +<%inherit file="../main.html" /> +<%namespace name='static' file='../static_content.html'/> +<%block name="bodyclass">discussion +<%block name="title">Discussion – MITx 6.002x + +<%block name="headextra"> +<%static:css group='course'/> + + +<%block name="js_extra"> +<%include file="_js_dependencies.html" /> + + +<%include file="../course_navigation.html" args="active_page='discussion'" /> + +
+
+ + +
+ ${content} +
+
+
+ + diff --git a/lms/templates/grade_summary.html b/lms/templates/grade_summary.html new file mode 100644 index 0000000000..3fdcd910ae --- /dev/null +++ b/lms/templates/grade_summary.html @@ -0,0 +1,16 @@ +<%inherit file="main.html" /> +<%! from django.core.urlresolvers import reverse %> +<%namespace name='static' file='static_content.html'/> + +<%include file="course_navigation.html" args="active_page=''" /> + +
+
+
+

Grade summary

+ +

Not implemented yet

+ +
+
+
diff --git a/lms/templates/gradebook.html b/lms/templates/gradebook.html index bdc1516339..787fc23c9a 100644 --- a/lms/templates/gradebook.html +++ b/lms/templates/gradebook.html @@ -1,15 +1,17 @@ <%inherit file="main.html" /> +<%! from django.core.urlresolvers import reverse %> <%namespace name='static' file='static_content.html'/> <%block name="js_extra"> + <%block name="headextra"> <%static:css group='course'/> - + - + + + <%include file="course_navigation.html" args="active_page=''" /> @@ -26,48 +34,76 @@

Gradebook

- - %if len(students) > 0: - - <% - templateSummary = students[0]['grade_summary'] - %> - - - - - %for section in templateSummary['section_breakdown']: - + +
Student${section['label']}
+ + + + + + + %for student in students: + + + %endfor - - - - <%def name="percent_data(percentage)"> - <% - letter_grade = 'None' - if percentage > 0: - letter_grade = 'F' - for grade in ['A', 'B', 'C']: - if percentage >= course.grade_cutoffs[grade]: - letter_grade = grade - break - - data_class = "grade_" + letter_grade - %> - - - - %for student in students: - - - %for section in student['grade_summary']['section_breakdown']: - ${percent_data( section['percent'] )} - %endfor - - - %endfor +
+
+ +
+
+ ${student['username']} +
Total
${ "{0:.0%}".format( percentage ) }
${student['username']}${percent_data( student['grade_summary']['percent'])}
+ + + + %if len(students) > 0: +
+ + <% + templateSummary = students[0]['grade_summary'] + %> + + + %for section in templateSummary['section_breakdown']: + + %endfor + + + + + <%def name="percent_data(fraction)"> + <% + letter_grade = 'None' + if fraction > 0: + letter_grade = 'F' + for grade in ['A', 'B', 'C']: + if fraction >= course.grade_cutoffs[grade]: + letter_grade = grade + break + + data_class = "grade_" + letter_grade + %> + + + + + %for student in students: + + %for section in student['grade_summary']['section_breakdown']: + ${percent_data( section['percent'] )} + %endfor + + + %endfor + +
${section['label']}Total
${ "{0:.0f}".format( 100 * fraction ) }
${percent_data( student['grade_summary']['percent'])}
+
+ %endif
+ + diff --git a/lms/templates/index.html b/lms/templates/index.html index 4255251604..0ee00b57c0 100644 --- a/lms/templates/index.html +++ b/lms/templates/index.html @@ -14,7 +14,7 @@
- Sign Up + Sign Up
@@ -148,7 +148,7 @@ % if show_signup_immediately is not UNDEFINED: + + + diff --git a/lms/templates/staticbook.html b/lms/templates/staticbook.html index 804676de97..2a0a6f03bb 100644 --- a/lms/templates/staticbook.html +++ b/lms/templates/staticbook.html @@ -75,7 +75,14 @@ $("#open_close_accordion a").click(function(){ <%def name="print_entry(entry)">
  • - ${' '.join(entry.get(attribute, '') for attribute in ['chapter', 'name', 'page_label'])} + + %if entry.get('chapter'): + ${entry.get('chapter')}. ${entry.get('name')} + %else: + ${entry.get('name')} + %endif + + ${entry.get('page_label')} % if len(entry) > 0:
      diff --git a/lms/urls.py b/lms/urls.py index 5def025a30..fe8bf0193b 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -2,7 +2,6 @@ from django.conf import settings from django.conf.urls import patterns, include, url from django.contrib import admin from django.conf.urls.static import static - import django.contrib.auth.views # Uncomment the next two lines to enable the admin: @@ -15,7 +14,7 @@ urlpatterns = ('', url(r'^dashboard$', 'student.views.dashboard', name="dashboard"), url(r'^admin_dashboard$', 'dashboard.views.dashboard'), - + url(r'^change_email$', 'student.views.change_email_request'), url(r'^email_confirm/(?P[^/]*)$', 'student.views.confirm_email_change'), url(r'^change_name$', 'student.views.change_name_request'), @@ -85,7 +84,6 @@ urlpatterns = ('', (r'^pressrelease$', 'django.views.generic.simple.redirect_to', {'url': '/press/uc-berkeley-joins-edx'}), - (r'^favicon\.ico$', 'django.views.generic.simple.redirect_to', {'url': '/static/images/favicon.ico'}), # TODO: These urls no longer work. They need to be updated before they are re-enabled @@ -98,12 +96,18 @@ if settings.PERFSTATS: if settings.COURSEWARE_ENABLED: urlpatterns += ( + # Hook django-masquerade, allowing staff to view site as other users url(r'^masquerade/', include('masquerade.urls')), url(r'^jump_to/(?P.*)$', 'courseware.views.jump_to', name="jump_to"), - url(r'^modx/(?P.*?)/(?P[^/]*)$', 'courseware.module_render.modx_dispatch'), #reset_problem'), - url(r'^xqueue/(?P[^/]*)/(?P.*?)/(?P[^/]*)$', 'courseware.module_render.xqueue_callback'), - url(r'^change_setting$', 'student.views.change_setting'), + url(r'^courses/(?P[^/]+/[^/]+/[^/]+)/modx/(?P.*?)/(?P[^/]*)$', + 'courseware.module_render.modx_dispatch', + name='modx_dispatch'), + url(r'^courses/(?P[^/]+/[^/]+/[^/]+)/xqueue/(?P[^/]*)/(?P.*?)/(?P[^/]*)$', + 'courseware.module_render.xqueue_callback', + name='xqueue_callback'), + url(r'^change_setting$', 'student.views.change_setting', + name='change_setting'), # TODO: These views need to be updated before they work # url(r'^calculate$', 'util.views.calculate'), @@ -118,7 +122,7 @@ if settings.COURSEWARE_ENABLED: #About the course url(r'^courses/(?P[^/]+/[^/]+/[^/]+)/about$', 'courseware.views.course_about', name="about_course"), - + #Inside the course url(r'^courses/(?P[^/]+/[^/]+/[^/]+)/info$', 'courseware.views.course_info', name="info"), @@ -130,22 +134,37 @@ if settings.COURSEWARE_ENABLED: 'staticbook.views.index_shifted'), url(r'^courses/(?P[^/]+/[^/]+/[^/]+)/courseware/?$', 'courseware.views.index', name="courseware"), + url(r'^courses/(?P[^/]+/[^/]+/[^/]+)/courseware/(?P[^/]*)/$', + 'courseware.views.index', name="courseware_chapter"), url(r'^courses/(?P[^/]+/[^/]+/[^/]+)/courseware/(?P[^/]*)/(?P
      [^/]*)/$', 'courseware.views.index', name="courseware_section"), url(r'^courses/(?P[^/]+/[^/]+/[^/]+)/profile$', 'courseware.views.profile', name="profile"), + # Takes optional student_id for instructor use--shows profile as that student sees it. url(r'^courses/(?P[^/]+/[^/]+/[^/]+)/profile/(?P[^/]*)/$', - 'courseware.views.profile'), - url(r'^courses/(?P[^/]+/[^/]+/[^/]+)/news$', - 'courseware.views.news', name="news"), + 'courseware.views.profile', name="student_profile"), + + # For the instructor + url(r'^courses/(?P[^/]+/[^/]+/[^/]+)/instructor$', + 'courseware.views.instructor_dashboard', name="instructor_dashboard"), - # discussion - url(r'^courses/(?P[^/]+/[^/]+/[^/]+)/discussion/', - include('django_comment_client.urls')), url(r'^courses/(?P[^/]+/[^/]+/[^/]+)/gradebook$', - 'courseware.views.gradebook'), + 'courseware.views.gradebook', name='gradebook'), + url(r'^courses/(?P[^/]+/[^/]+/[^/]+)/grade_summary$', + 'courseware.views.grade_summary', name='grade_summary'), + ) + # discussion forums live within courseware, so courseware must be enabled first + if settings.DISCUSSION_SERVICE_ENABLED: + + urlpatterns += ( + url(r'^courses/(?P[^/]+/[^/]+/[^/]+)/news$', + 'courseware.views.news', name="news"), + url(r'^courses/(?P[^/]+/[^/]+/[^/]+)/discussion/', + include('django_comment_client.urls')) + ) + # Multicourse wiki if settings.WIKI_ENABLED: urlpatterns += ( @@ -166,6 +185,7 @@ if settings.ASKBOT_ENABLED: ) + if settings.DEBUG: ## Jasmine urlpatterns=urlpatterns + (url(r'^_jasmine/', include('django_jasmine.urls')),) diff --git a/proxy/nginx.conf b/proxy/nginx.conf new file mode 100644 index 0000000000..470c3933ac --- /dev/null +++ b/proxy/nginx.conf @@ -0,0 +1,67 @@ +# Mapping of +# +# From the /mitx directory: +# /usr/local/Cellar/nginx/1.2.2/sbin/nginx -p `pwd`/ -c nginx.conf + +worker_processes 1; + +events { + worker_connections 1024; +} + +http { + ## + # Basic Settings + ## + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + # server_tokens off; + # server_names_hash_bucket_size 64; + # server_name_in_redirect off; + + include /usr/local/etc/nginx/mime.types; + default_type application/octet-stream; + + ## + # Gzip Settings + ## + gzip on; + gzip_disable "msie6"; + + upstream portal { + server localhost:8000; + } + + upstream course_harvardx_cs50_2012 { + server localhost:8001; + } + + upstream course_mitx_6002_2012_fall { + server localhost:8002; + } + + # Mostly copied from our existing server... + server { + listen 8100 default_server; + + rewrite ^(.*)/favicon.ico$ /static/images/favicon.ico last; + + # Our catchall + location / { + proxy_pass http://portal; + } + + location /courses/HarvardX/CS50x/2012/ { + proxy_pass http://course_harvardx_cs50_2012; + } + + location /courses/MITx/6.002x/2012_Fall/ { + proxy_pass http://course_mitx_6002_2012_fall; + } + } +} + + diff --git a/rakefile b/rakefile index caf0d58f2f..c62c87701e 100644 --- a/rakefile +++ b/rakefile @@ -88,7 +88,8 @@ $failed_tests = 0 def run_tests(system, report_dir, stop_on_failure=true) ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml") ENV['NOSE_COVER_HTML_DIR'] = File.join(report_dir, "cover") - sh(django_admin(system, :test, 'test', *Dir["#{system}/djangoapps/*"].each)) do |ok, res| + dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"] + sh(django_admin(system, :test, 'test', *dirs.each)) do |ok, res| if !ok and stop_on_failure abort "Test failed!" end diff --git a/run_watch_data.py b/run_watch_data.py index f5605a5c6a..c6cdd4f0df 100755 --- a/run_watch_data.py +++ b/run_watch_data.py @@ -17,7 +17,7 @@ from watchdog.events import LoggingEventHandler, FileSystemEventHandler # watch fewer or more extensions, you can change EXTENSIONS. To watch all # extensions, add "*" to EXTENSIONS. -WATCH_DIRS = ["../data"] +WATCH_DIRS = ["../data", "common/lib/xmodule/xmodule/js"] EXTENSIONS = ["*", "xml", "js", "css", "coffee", "scss", "html"] WATCH_DIRS = [os.path.abspath(os.path.normpath(dir)) for dir in WATCH_DIRS] diff --git a/utility-scripts/create_groups.py b/utility-scripts/create_groups.py index 33c563127f..0e3245bb4d 100644 --- a/utility-scripts/create_groups.py +++ b/utility-scripts/create_groups.py @@ -18,6 +18,7 @@ except Exception as err: from django.conf import settings from django.contrib.auth.models import User, Group from path import path +from lxml import etree data_dir = settings.DATA_DIR print "data_dir = %s" % data_dir @@ -26,7 +27,17 @@ for course_dir in os.listdir(data_dir): # print course_dir if not os.path.isdir(path(data_dir) / course_dir): continue - gname = 'staff_%s' % course_dir + + cxfn = path(data_dir) / course_dir / 'course.xml' + coursexml = etree.parse(cxfn) + cxmlroot = coursexml.getroot() + course = cxmlroot.get('course') + if course is None: + print "oops, can't get course id for %s" % course_dir + continue + print "course=%s for course_dir=%s" % (course,course_dir) + + gname = 'staff_%s' % course if Group.objects.filter(name=gname): print "group exists for %s" % gname continue