diff --git a/cms/djangoapps/contentstore/tests/test_import.py b/cms/djangoapps/contentstore/tests/test_import.py index 41b26b5501..77bdd05527 100644 --- a/cms/djangoapps/contentstore/tests/test_import.py +++ b/cms/djangoapps/contentstore/tests/test_import.py @@ -179,3 +179,24 @@ class ContentStoreImportTest(ModuleStoreTestCase): u'i4x://testX/peergrading_copy/combinedopenended/SampleQuestion', peergrading_module.link_to_location ) + + def test_rewrite_reference_value_dict(self): + module_store = modulestore('direct') + target_location = Location(['i4x', 'testX', 'split_test_copy', 'course', 'copy_run']) + import_from_xml( + module_store, + 'common/test/data/', + ['split_test_module'], + target_location_namespace=target_location + ) + split_test_module = module_store.get_item( + Location(['i4x', 'testX', 'split_test_copy', 'split_test', 'split1']) + ) + self.assertIsNotNone(split_test_module) + self.assertEqual( + { + "0": "i4x://testX/split_test_copy/vertical/sample_0", + "2": "i4x://testX/split_test_copy/vertical/sample_2", + }, + split_test_module.group_id_to_child, + ) diff --git a/cms/envs/common.py b/cms/envs/common.py index 9bba1f986f..b914239585 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -34,8 +34,7 @@ from path import path from lms.lib.xblock.mixin import LmsBlockMixin from cms.lib.xblock.mixin import CmsBlockMixin from xmodule.modulestore.inheritance import InheritanceMixin -from xmodule.modulestore import only_xmodules -from xmodule.x_module import XModuleMixin +from xmodule.x_module import XModuleMixin, prefer_xmodules from dealer.git import git ############################ FEATURE CONFIGURATION ############################# @@ -223,15 +222,10 @@ X_FRAME_OPTIONS = 'ALLOW' # once the responsibility of XBlock creation is moved out of modulestore - cpennington XBLOCK_MIXINS = (LmsBlockMixin, CmsBlockMixin, InheritanceMixin, XModuleMixin) -# Only allow XModules in Studio -XBLOCK_SELECT_FUNCTION = only_xmodules - -# Use the following lines to allow any xblock in Studio, -# either by uncommenting them here, or adding them to your private.py +# Allow any XBlock in Studio # You should also enable the ALLOW_ALL_ADVANCED_COMPONENTS feature flag, so that # xblocks can be added via advanced settings -# from xmodule.modulestore import prefer_xmodules -# XBLOCK_SELECT_FUNCTION = prefer_xmodules +XBLOCK_SELECT_FUNCTION = prefer_xmodules ############################ SIGNAL HANDLERS ################################ # This is imported to register the exception signal handling that logs exceptions @@ -473,6 +467,9 @@ INSTALLED_APPS = ( # for course creator table 'django.contrib.admin', + # XBlocks containing migrations + 'mentoring', + # for managing course modes 'course_modes', diff --git a/common/djangoapps/user_api/middleware.py b/common/djangoapps/user_api/middleware.py new file mode 100644 index 0000000000..3a33db7143 --- /dev/null +++ b/common/djangoapps/user_api/middleware.py @@ -0,0 +1,50 @@ +""" +Middleware for user api. +Adds user's tags to tracking event context. +""" +from track.contexts import COURSE_REGEX +from eventtracking import tracker +from user_api.models import UserCourseTag + + +class UserTagsEventContextMiddleware(object): + """Middleware that adds a user's tags to tracking event context.""" + CONTEXT_NAME = 'user_tags_context' + + def process_request(self, request): + """ + Add a user's tags to the tracking event context. + """ + match = COURSE_REGEX.match(request.build_absolute_uri()) + course_id = None + if match: + course_id = match.group('course_id') + + context = {} + + if course_id: + context['course_id'] = course_id + + if request.user.is_authenticated(): + context['course_user_tags'] = dict( + UserCourseTag.objects.filter( + user=request.user.pk, + course_id=course_id + ).values_list('key', 'value') + ) + else: + context['course_user_tags'] = {} + + tracker.get_tracker().enter_context( + self.CONTEXT_NAME, + context + ) + + def process_response(self, request, response): # pylint: disable=unused-argument + """Exit the context if it exists.""" + try: + tracker.get_tracker().exit_context(self.CONTEXT_NAME) + except: # pylint: disable=bare-except + pass + + return response diff --git a/common/djangoapps/user_api/migrations/0002_auto__add_usercoursetags__add_unique_usercoursetags_user_course_id_key.py b/common/djangoapps/user_api/migrations/0002_auto__add_usercoursetags__add_unique_usercoursetags_user_course_id_key.py new file mode 100644 index 0000000000..cc7e27cc4d --- /dev/null +++ b/common/djangoapps/user_api/migrations/0002_auto__add_usercoursetags__add_unique_usercoursetags_user_course_id_key.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'UserCourseTags' + db.create_table('user_api_usercoursetags', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('user', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', to=orm['auth.User'])), + ('key', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)), + ('course_id', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)), + ('value', self.gf('django.db.models.fields.TextField')()), + )) + db.send_create_signal('user_api', ['UserCourseTags']) + + # Adding unique constraint on 'UserCourseTags', fields ['user', 'course_id', 'key'] + db.create_unique('user_api_usercoursetags', ['user_id', 'course_id', 'key']) + + + def backwards(self, orm): + # Removing unique constraint on 'UserCourseTags', fields ['user', 'course_id', 'key'] + db.delete_unique('user_api_usercoursetags', ['user_id', 'course_id', 'key']) + + # Deleting model 'UserCourseTags' + db.delete_table('user_api_usercoursetags') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'user_api.usercoursetags': { + 'Meta': {'unique_together': "(('user', 'course_id', 'key'),)", 'object_name': 'UserCourseTags'}, + 'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['auth.User']"}), + 'value': ('django.db.models.fields.TextField', [], {}) + }, + 'user_api.userpreference': { + 'Meta': {'unique_together': "(('user', 'key'),)", 'object_name': 'UserPreference'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['auth.User']"}), + 'value': ('django.db.models.fields.TextField', [], {}) + } + } + + complete_apps = ['user_api'] \ No newline at end of file diff --git a/common/djangoapps/user_api/migrations/0003_rename_usercoursetags.py b/common/djangoapps/user_api/migrations/0003_rename_usercoursetags.py new file mode 100644 index 0000000000..dc448817e3 --- /dev/null +++ b/common/djangoapps/user_api/migrations/0003_rename_usercoursetags.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.rename_table('user_api_usercoursetags', 'user_api_usercoursetag') + + + def backwards(self, orm): + db.rename_table('user_api_usercoursetag', 'user_api_usercoursetags') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'user_api.usercoursetag': { + 'Meta': {'unique_together': "(('user', 'course_id', 'key'),)", 'object_name': 'UserCourseTag'}, + 'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['auth.User']"}), + 'value': ('django.db.models.fields.TextField', [], {}) + }, + 'user_api.userpreference': { + 'Meta': {'unique_together': "(('user', 'key'),)", 'object_name': 'UserPreference'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['auth.User']"}), + 'value': ('django.db.models.fields.TextField', [], {}) + } + } + + complete_apps = ['user_api'] \ No newline at end of file diff --git a/common/djangoapps/user_api/models.py b/common/djangoapps/user_api/models.py index 77ab4c6c95..36ed30eddc 100644 --- a/common/djangoapps/user_api/models.py +++ b/common/djangoapps/user_api/models.py @@ -8,7 +8,7 @@ class UserPreference(models.Model): key = models.CharField(max_length=255, db_index=True) value = models.TextField() - class Meta: + class Meta: # pylint: disable=missing-docstring unique_together = ("user", "key") @classmethod @@ -33,3 +33,17 @@ class UserPreference(models.Model): return user_pref.value except cls.DoesNotExist: return default + + +class UserCourseTag(models.Model): + """ + Per-course user tags, to be used by various things that want to store tags about + the user. Added initially to store assignment to experimental groups. + """ + user = models.ForeignKey(User, db_index=True, related_name="+") + key = models.CharField(max_length=255, db_index=True) + course_id = models.CharField(max_length=255, db_index=True) + value = models.TextField() + + class Meta: # pylint: disable=missing-docstring + unique_together = ("user", "course_id", "key") diff --git a/common/djangoapps/user_api/tests/factories.py b/common/djangoapps/user_api/tests/factories.py index ee2609cb5b..535e888a59 100644 --- a/common/djangoapps/user_api/tests/factories.py +++ b/common/djangoapps/user_api/tests/factories.py @@ -1,10 +1,23 @@ +"""Provides factories for User API models.""" from factory.django import DjangoModelFactory -from user_api.models import UserPreference - +from factory import SubFactory +from student.tests.factories import UserFactory +from user_api.models import UserPreference, UserCourseTag +# Factories don't have __init__ methods, and are self documenting +# pylint: disable=W0232, C0111 class UserPreferenceFactory(DjangoModelFactory): FACTORY_FOR = UserPreference user = None key = None value = "default test value" + + +class UserCourseTagFactory(DjangoModelFactory): + FACTORY_FOR = UserCourseTag + + user = SubFactory(UserFactory) + course_id = 'org/course/run' + key = None + value = None diff --git a/common/djangoapps/user_api/tests/test_middleware.py b/common/djangoapps/user_api/tests/test_middleware.py new file mode 100644 index 0000000000..290ca22048 --- /dev/null +++ b/common/djangoapps/user_api/tests/test_middleware.py @@ -0,0 +1,119 @@ +"""Tests for user API middleware""" +from mock import Mock, patch +from unittest import TestCase + +from django.http import HttpResponse +from django.test.client import RequestFactory + +from student.tests.factories import UserFactory, AnonymousUserFactory +from user_api.tests.factories import UserCourseTagFactory +from user_api.middleware import UserTagsEventContextMiddleware + + +class TagsMiddlewareTest(TestCase): + """ + Test the UserTagsEventContextMiddleware + """ + def setUp(self): + self.middleware = UserTagsEventContextMiddleware() + self.user = UserFactory.create() + self.other_user = UserFactory.create() + + self.course_id = 'mock/course/id' + self.request_factory = RequestFactory() + + # TODO: Make it so we can use reverse. Appears to fail depending on the order in which tests are run + #self.request = RequestFactory().get(reverse('courseware', kwargs={'course_id': self.course_id})) + self.request = RequestFactory().get('/courses/{}/courseware'.format(self.course_id)) + self.request.user = self.user + + self.response = Mock(spec=HttpResponse) + + patcher = patch('user_api.middleware.tracker') + self.tracker = patcher.start() + self.addCleanup(patcher.stop) + + def process_request(self): + """ + Execute process request using the request, and verify that it returns None + so that the request continues. + """ + # Middleware should pass request through + self.assertEquals(self.middleware.process_request(self.request), None) + + def assertContextSetTo(self, context): + """Asserts UserTagsEventContextMiddleware.CONTEXT_NAME matches ``context``""" + self.tracker.get_tracker.return_value.enter_context.assert_called_with( # pylint: disable=maybe-no-member + UserTagsEventContextMiddleware.CONTEXT_NAME, + context + ) + + def test_tag_context(self): + for key, value in (('int_value', 1), ('str_value', "two")): + UserCourseTagFactory.create( + course_id=self.course_id, + user=self.user, + key=key, + value=value, + ) + + UserCourseTagFactory.create( + course_id=self.course_id, + user=self.other_user, + key="other_user", + value="other_user_value" + ) + + UserCourseTagFactory.create( + course_id='other/course/id', + user=self.user, + key="other_course", + value="other_course_value" + ) + + self.process_request() + self.assertContextSetTo({ + 'course_id': self.course_id, + 'course_user_tags': { + 'int_value': '1', + 'str_value': 'two', + } + }) + + def test_no_tags(self): + self.process_request() + self.assertContextSetTo({'course_id': self.course_id, 'course_user_tags': {}}) + + def test_not_course_url(self): + self.request = self.request_factory.get('/not/a/course/url') + self.request.user = self.user + + self.process_request() + + self.assertContextSetTo({}) + + def test_anonymous_user(self): + self.request.user = AnonymousUserFactory() + + self.process_request() + + self.assertContextSetTo({'course_id': self.course_id, 'course_user_tags': {}}) + + def test_remove_context(self): + get_tracker = self.tracker.get_tracker # pylint: disable=maybe-no-member + exit_context = get_tracker.return_value.exit_context + + # The middleware should clean up the context when the request is done + self.assertEquals( + self.middleware.process_response(self.request, self.response), + self.response + ) + exit_context.assert_called_with(UserTagsEventContextMiddleware.CONTEXT_NAME) + exit_context.reset_mock() + + # Even if the tracker blows up, the middleware should still return the response + get_tracker.side_effect = Exception + self.assertEquals( + self.middleware.process_response(self.request, self.response), + self.response + ) diff --git a/common/djangoapps/user_api/tests/test_user_service.py b/common/djangoapps/user_api/tests/test_user_service.py new file mode 100644 index 0000000000..f63f702bcb --- /dev/null +++ b/common/djangoapps/user_api/tests/test_user_service.py @@ -0,0 +1,34 @@ +""" +Test the user service +""" +from django.test import TestCase + +from student.tests.factories import UserFactory +from user_api import user_service + + +class TestUserService(TestCase): + """ + Test the user service + """ + def setUp(self): + self.user = UserFactory.create() + self.course_id = 'test_org/test_course_number/test_run' + self.test_key = 'test_key' + + def test_get_set_course_tag(self): + # get a tag that doesn't exist + tag = user_service.get_course_tag(self.user, self.course_id, self.test_key) + self.assertIsNone(tag) + + # test setting a new key + test_value = 'value' + user_service.set_course_tag(self.user, self.course_id, self.test_key, test_value) + tag = user_service.get_course_tag(self.user, self.course_id, self.test_key) + self.assertEqual(tag, test_value) + + #test overwriting an existing key + test_value = 'value2' + user_service.set_course_tag(self.user, self.course_id, self.test_key, test_value) + tag = user_service.get_course_tag(self.user, self.course_id, self.test_key) + self.assertEqual(tag, test_value) diff --git a/common/djangoapps/user_api/user_service.py b/common/djangoapps/user_api/user_service.py new file mode 100644 index 0000000000..5bfe7fe6f0 --- /dev/null +++ b/common/djangoapps/user_api/user_service.py @@ -0,0 +1,65 @@ +""" +A service-like user_info interface. Could be made into an http API later, but for now +just in-process. Exposes global and per-course key-value pairs for users. + +Implementation note: +Stores global metadata using the UserPreference model, and per-course metadata using the +UserCourseTag model. +""" + +from user_api.models import UserCourseTag + +# Scopes +# (currently only allows per-course tags. Can be expanded to support +# global tags (e.g. using the existing UserPreferences table)) +COURSE_SCOPE = 'course' + +def get_course_tag(user, course_id, key): + """ + Gets the value of the user's course tag for the specified key in the specified + course_id. + + Args: + user: the User object for the course tag + course_id: course identifier (string) + key: arbitrary (<=255 char string) + + Returns: + string value, or None if there is no value saved + """ + try: + record = UserCourseTag.objects.get( + user=user, + course_id=course_id, + key=key) + + return record.value + except UserCourseTag.DoesNotExist: + return None + + +def set_course_tag(user, course_id, key, value): + """ + Sets the value of the user's course tag for the specified key in the specified + course_id. Overwrites any previous value. + + The intention is that the values are fairly short, as they will be included in all + analytics events about this user. + + Args: + user: the User object + course_id: course identifier (string) + key: arbitrary (<=255 char string) + value: arbitrary string + """ + + record, _ = UserCourseTag.objects.get_or_create( + user=user, + course_id=course_id, + key=key) + + record.value = value + record.save() + + # TODO: There is a risk of IntegrityErrors being thrown here given + # simultaneous calls from many processes. Handle by retrying after a short delay? diff --git a/common/lib/capa/capa/inputtypes.py b/common/lib/capa/capa/inputtypes.py index 790e93679d..d638b41350 100644 --- a/common/lib/capa/capa/inputtypes.py +++ b/common/lib/capa/capa/inputtypes.py @@ -46,6 +46,7 @@ import re import shlex # for splitting quoted strings import sys import pyparsing +import html5lib from .registry import TagRegistry from chem import chemcalc @@ -286,7 +287,18 @@ class InputTypeBase(object): context = self._get_render_context() html = self.capa_system.render_template(self.template, context) - return etree.XML(html) + + try: + output = etree.XML(html) + except etree.XMLSyntaxError as ex: + # If `html` contains attrs with no values, like `controls` in