split testing support in the LMS.

Adds a split_test_module XModule, that can choose one of its children
to display, based on a get_condition_for_user API added to the runtime.

To test, add something like this to an xml course, or make equivalent
 tweaks in mongo.

 <vertical url_name="split_test_vert">
    <split_test url_name="split1" experiment_id="0" condition_id_to_child='{"0": "i4x://MITx/6.00x/html/split_test_cond0", "1": "i4x://MITx/6.00x/html/split_test_cond1"}'>
       <html url_name="split_test_cond0">condition 0</html>
       <html url_name="split_test_cond1">condition 1</html>
    </split_test>
  </vertical>

Also needs an experiment configured in the course policy json: e.g.

        "user_partitions": [{"id": 0,
                                      "name": "Experiment 0",
                                      "description": "Unicorns?",
                                      "version": 1,
                                      "groups": [{"id": 0,
                                                        "name": "group 0",
                                                        "version": 1},
                                                        {"id": 1,
                                                        "name": "group 1",
                                                        "version": 1}]}]

(This particular snippet will work inside a course with org MITx
 and course name 6.00x)

Co-Author: Sarina Canelake <sarina@edx.org>
Co-Author: Julia Hansbrough <julia@edx.org>
Co-Author: Diana Huang <diana@edx.org>
Co-Author: Calen Pennington <cale@edx.org>

[LMS-2095]
This commit is contained in:
Victor Shnayder
2013-12-31 10:44:34 -05:00
committed by Calen Pennington
parent 72e876fe40
commit 281ad63d2b
22 changed files with 1229 additions and 6 deletions

View File

@@ -17,6 +17,7 @@ XMODULES = [
"problem = xmodule.capa_module:CapaDescriptor",
"problemset = xmodule.seq_module:SequenceDescriptor",
"randomize = xmodule.randomize_module:RandomizeDescriptor",
"split_test = xmodule.split_test_module:SplitTestDescriptor",
"section = xmodule.backcompat_module:SemanticSectionDescriptor",
"sequential = xmodule.seq_module:SequenceDescriptor",
"slides = xmodule.backcompat_module:TranslateCustomTagDescriptor",

View File

@@ -9,6 +9,7 @@ import dateutil.parser
from lazy import lazy
from xmodule.modulestore import Location
from xmodule.partitions.partitions import UserPartition
from xmodule.seq_module import SequenceDescriptor, SequenceModule
from xmodule.graders import grader_from_conf
import json
@@ -156,10 +157,27 @@ class TextbookList(List):
return json_data
class UserPartitionList(List):
def from_json(self, values):
return [UserPartition.from_json(v) for v in values]
def to_json(self, values):
return [user_partition.to_json()
for user_partition in values]
class CourseFields(object):
lti_passports = List(help="LTI tools passports as id:client_key:client_secret", scope=Scope.settings)
textbooks = TextbookList(help="List of pairs of (title, url) for textbooks used in this course",
default=[], scope=Scope.content)
# This field is intended for Studio to update, not to be exposed directly via
# advanced_settings.
user_partitions = UserPartitionList(
help="List of user partitions of this course into groups, used e.g. for experiments",
default=[], scope=Scope.content)
wiki_slug = String(help="Slug that points to the wiki for this course", scope=Scope.content)
enrollment_start = Date(help="Date that enrollment for this class is opened", scope=Scope.settings)
enrollment_end = Date(help="Date that enrollment for this class is closed", scope=Scope.settings)
@@ -354,7 +372,7 @@ class CourseFields(object):
# Ensure that courses imported from XML keep their image
default="images_course_image.jpg"
)
## Course level Certificate Name overrides.
cert_name_short = String(
help="Sitewide name of completion statements given to students (short).",

View File

@@ -321,13 +321,13 @@ class MongoModuleStore(ModuleStoreWriteBase):
'''
TODO (cdodge) This method can be deleted when the 'split module store' work has been completed
'''
# get all collections in the course, this query should not return any leaf nodes
# note this is a bit ugly as when we add new categories of containers, we have to add it here
block_types_with_children = set(name for name, class_ in XBlock.load_classes() if getattr(class_, 'has_children', False))
query = {'_id.org': location.org,
'_id.course': location.course,
'_id.category': {'$in': ['course', 'chapter', 'sequential', 'vertical', 'videosequence',
'wrapper', 'problemset', 'conditional', 'randomize']}
'_id.category': {'$in': list(block_types_with_children)}
}
# we just want the Location, children, and inheritable metadata
record_filter = {'_id': 1, 'definition.children': 1}

View File

@@ -0,0 +1,116 @@
"""Defines ``Group`` and ``UserPartition`` models for partitioning"""
# We use ``id`` in this file as the IDs of our Groups and UserPartitions,
# which Pylint disapproves of.
# pylint: disable=invalid-name, redefined-builtin
class Group(object):
"""
An id and name for a group of students. The id should be unique
within the UserPartition this group appears in.
"""
# in case we want to add to this class, a version will be handy
# for deserializing old versions. (This will be serialized in courses)
VERSION = 1
def __init__(self, id, name):
self.id = int(id)
self.name = name
def to_json(self):
"""
'Serialize' to a json-serializable representation.
Returns:
a dictionary with keys for the properties of the group.
"""
return {
"id": self.id,
"name": self.name,
"version": Group.VERSION
}
@staticmethod
def from_json(value):
"""
Deserialize a Group from a json-like representation.
Args:
value: a dictionary with keys for the properties of the group.
Raises TypeError if the value doesn't have the right keys.
"""
for key in ('id', 'name', 'version'):
if key not in value:
raise TypeError("Group dict {0} missing value key '{1}'".format(
value, key))
if value["version"] != Group.VERSION:
raise TypeError("Group dict {0} has unexpected version".format(
value))
return Group(value["id"], value["name"])
class UserPartition(object):
"""
A named way to partition users into groups, primarily intended for running
experiments. It is expected that each user will be in at most one group in a
partition.
A Partition has an id, name, description, and a list of groups.
The id is intended to be unique within the context where these are used. (e.g. for
partitions of users within a course, the ids should be unique per-course)
"""
VERSION = 1
def __init__(self, id, name, description, groups):
self.id = int(id)
self.name = name
self.description = description
self.groups = groups
def to_json(self):
"""
'Serialize' to a json-serializable representation.
Returns:
a dictionary with keys for the properties of the partition.
"""
return {
"id": self.id,
"name": self.name,
"description": self.description,
"groups": [g.to_json() for g in self.groups],
"version": UserPartition.VERSION
}
@staticmethod
def from_json(value):
"""
Deserialize a Group from a json-like representation.
Args:
value: a dictionary with keys for the properties of the group.
Raises TypeError if the value doesn't have the right keys.
"""
for key in ('id', 'name', 'description', 'version', 'groups'):
if key not in value:
raise TypeError("UserPartition dict {0} missing value key '{1}'"
.format(value, key))
if value["version"] != UserPartition.VERSION:
raise TypeError("UserPartition dict {0} has unexpected version"
.format(value))
groups = [Group.from_json(g) for g in value["groups"]]
return UserPartition(
value["id"],
value["name"],
value["description"],
groups
)

View File

@@ -0,0 +1,138 @@
"""
This is a service-like API that assigns tracks which groups users are in for various
user partitions. It uses the user_service key/value store provided by the LMS runtime to
persist the assignments.
"""
import random
from abc import ABCMeta, abstractproperty
class PartitionService(object):
"""
This is an XBlock service that assigns tracks which groups users are in for various
user partitions. It uses the provided user_tags service object to
persist the assignments.
"""
__metaclass__ = ABCMeta
@abstractproperty
def course_partitions(self):
"""
Return the set of partitions assigned to self._course_id
"""
raise NotImplementedError('Subclasses must implement course_partition')
def __init__(self, user_tags_service, course_id, track_function):
self.random = random.Random()
self._user_tags_service = user_tags_service
self._course_id = course_id
self._track_function = track_function
def get_user_group_for_partition(self, user_partition_id):
"""
If the user is already assigned to a group in user_partition_id, return the
group_id.
If not, assign them to one of the groups, persist that decision, and
return the group_id.
If the group they are assigned to doesn't exist anymore, re-assign to one of
the existing groups and return its id.
Args:
user_partition_id -- an id of a partition that's hopefully in the
runtime.user_partitions list.
Returns:
The id of one of the groups in the specified user_partition_id (as a string).
Raises:
ValueError if the user_partition_id isn't found.
"""
user_partition = self._get_user_partition(user_partition_id)
if user_partition is None:
raise ValueError(
"Configuration problem! No user_partition with id {0} "
"in course {1}".format(user_partition_id, self._course_id)
)
group_id = self._get_group(user_partition)
return group_id
def _get_user_partition(self, user_partition_id):
"""
Look for a user partition with a matching id in
in the course's partitions.
Returns:
A UserPartition, or None if not found.
"""
for partition in self.course_partitions:
if partition.id == user_partition_id:
return partition
return None
def _key_for_partition(self, user_partition):
"""
Returns the key to use to look up and save the user's group for a particular
condition. Always use this function rather than constructing the key directly.
"""
return 'xblock.partition_service.partition_{0}'.format(user_partition.id)
def _get_group(self, user_partition):
"""
Return the group of the current user in user_partition. If they don't already have
one assigned, pick one and save it. Uses the runtime's user_service service to look up
and persist the info.
"""
key = self._key_for_partition(user_partition)
scope = self._user_tags_service.COURSE
group_id = self._user_tags_service.get_tag(scope, key)
if group_id is not None:
group_id = int(group_id)
partition_group_ids = [group.id for group in user_partition.groups]
# If a valid group id has been saved already, return it
if group_id is not None and group_id in partition_group_ids:
return group_id
# TODO: what's the atomicity of the get above and the save here? If it's not in a
# single transaction, we could get a situation where the user sees one state in one
# thread, but then that decision gets overwritten--low probability, but still bad.
# (If it is truly atomic, we should be fine--if one process is in the
# process of finding no group and making one, the other should block till it
# appears. HOWEVER, if we allow reads by the second one while the first
# process runs the transaction, we have a problem again: could read empty,
# have the first transaction finish, and pick a different group in a
# different process.)
# If a group id hasn't yet been saved, or the saved group id is invalid,
# we need to pick one, save it, then return it
# TODO: had a discussion in arch council about making randomization more
# deterministic (e.g. some hash). Could do that, but need to be careful not
# to introduce correlation between users or bias in generation.
# See note above for explanation of local_random()
group = self.random.choice(user_partition.groups)
self._user_tags_service.set_tag(scope, key, group.id)
# emit event for analytics
# FYI - context is always user ID that is logged in, NOT the user id that is
# being operated on. If instructor can move user explicitly, then we should
# put in event_info the user id that is being operated on.
event_info = {
'group_id': group.id,
'group_name': group.name,
'partition_id': user_partition.id,
'partition_name': user_partition.name
}
# TODO: Use the XBlock publish api instead
self._track_function('edx.split_test.assigned_user_to_partition', event_info)
return group.id

View File

@@ -0,0 +1,155 @@
"""
Test the partitions and partitions service
"""
from unittest import TestCase
from mock import Mock, MagicMock
from xmodule.partitions.partitions import Group, UserPartition
from xmodule.partitions.partitions_service import PartitionService
class TestGroup(TestCase):
"""Test constructing groups"""
def test_construct(self):
test_id = 10
name = "Grendel"
group = Group(test_id, name)
self.assertEqual(group.id, test_id)
self.assertEqual(group.name, name)
def test_string_id(self):
test_id = "10"
name = "Grendel"
group = Group(test_id, name)
self.assertEqual(group.id, 10)
def test_to_json(self):
test_id = 10
name = "Grendel"
group = Group(test_id, name)
jsonified = group.to_json()
act_jsonified = {
"id": test_id,
"name": name,
"version": group.VERSION
}
self.assertEqual(jsonified, act_jsonified)
def test_from_json(self):
test_id = 5
name = "Grendel"
jsonified = {
"id": test_id,
"name": name,
"version": Group.VERSION
}
group = Group.from_json(jsonified)
self.assertEqual(group.id, test_id)
self.assertEqual(group.name, name)
class StaticPartitionService(PartitionService):
"""
Mock PartitionService for testing.
"""
def __init__(self, partitions, **kwargs):
super(StaticPartitionService, self).__init__(**kwargs)
self._partitions = partitions
@property
def course_partitions(self):
return self._partitions
class TestPartitionsService(TestCase):
"""
Test getting a user's group out of a partition
"""
def setUp(self):
groups = [Group(0, 'Group 1'), Group(1, 'Group 2')]
self.partition_id = 0
# construct the user_service
self.user_tags = dict()
self.user_tags_service = MagicMock()
def mock_set_tag(_scope, key, value):
"""Sets the value of ``key`` to ``value``"""
self.user_tags[key] = value
def mock_get_tag(_scope, key):
"""Gets the value of ``key``"""
if key in self.user_tags:
return self.user_tags[key]
return None
self.user_tags_service.set_tag = mock_set_tag
self.user_tags_service.get_tag = mock_get_tag
user_partition = UserPartition(self.partition_id, 'Test Partition', 'for testing purposes', groups)
self.partitions_service = StaticPartitionService(
[user_partition],
user_tags_service=self.user_tags_service,
course_id=Mock(),
track_function=Mock()
)
def test_get_user_group_for_partition(self):
# get a group assigned to the user
group1 = self.partitions_service.get_user_group_for_partition(self.partition_id)
# make sure we get the same group back out if we try a second time
group2 = self.partitions_service.get_user_group_for_partition(self.partition_id)
self.assertEqual(group1, group2)
# test that we error if given an invalid partition id
with self.assertRaises(ValueError):
self.partitions_service.get_user_group_for_partition(3)
def test_user_in_deleted_group(self):
# get a group assigned to the user - should be group 0 or 1
old_group = self.partitions_service.get_user_group_for_partition(self.partition_id)
self.assertIn(old_group, [0, 1])
# Change the group definitions! No more group 0 or 1
groups = [Group(3, 'Group 3'), Group(4, 'Group 4')]
user_partition = UserPartition(self.partition_id, 'Test Partition', 'for testing purposes', groups)
self.partitions_service = StaticPartitionService(
[user_partition],
user_tags_service=self.user_tags_service,
course_id=Mock(),
track_function=Mock()
)
# Now, get a new group using the same call - should be 3 or 4
new_group = self.partitions_service.get_user_group_for_partition(self.partition_id)
self.assertIn(new_group, [3, 4])
# We should get the same group over multiple calls
new_group_2 = self.partitions_service.get_user_group_for_partition(self.partition_id)
self.assertEqual(new_group, new_group_2)
def test_change_group_name(self):
# Changing the name of the group shouldn't affect anything
# get a group assigned to the user - should be group 0 or 1
old_group = self.partitions_service.get_user_group_for_partition(self.partition_id)
self.assertIn(old_group, [0, 1])
# Change the group names
groups = [Group(0, 'Group 0'), Group(1, 'Group 1')]
user_partition = UserPartition(self.partition_id, 'Test Partition', 'for testing purposes', groups)
self.partitions_service = StaticPartitionService(
[user_partition],
user_tags_service=self.user_tags_service,
course_id=Mock(),
track_function=Mock()
)
# Now, get a new group using the same call
new_group = self.partitions_service.get_user_group_for_partition(self.partition_id)
self.assertEqual(old_group, new_group)

View File

@@ -0,0 +1,43 @@
/**
* Creates a new selector for managing toggling which child to show
* @constructor
*/
function ABTestSelector(elem) {
me = this;
me.elem = $(elem);
select_child = function(group_id) {
// iterate over all the children and hide all the ones that haven't been selected
// and show the one that was selected
me.elem.find('.split-test-child').each(function() {
// force this id to remain a string, even if it looks like something else
child_group_id = $(this).data('group-id').toString();
if(child_group_id === group_id) {
$(this).show();
}
else {
$(this).hide();
}
});
}
// hide all the children
me.elem.find('.split-test-child').hide();
select = me.elem.find('.split-test-select');
cur_group_id = select.val();
select_child(cur_group_id);
// bind the change event to the dropdown
select.change(function() {
group_id = $(this).val()
select_child(group_id);
});
}

View File

@@ -0,0 +1,185 @@
"""
Module for running content split tests
"""
import logging
from xmodule.progress import Progress
from xmodule.seq_module import SequenceDescriptor
from xmodule.x_module import XModule, module_attr
from lxml import etree
from xblock.core import XBlock
from xblock.fields import Scope, Integer, Dict
from xblock.fragment import Fragment
log = logging.getLogger('edx.' + __name__)
class SplitTestFields(object):
user_partition_id = Integer(help="Which user partition is used for this test",
scope=Scope.content)
# group_id is an int
# child is a serialized UsageId (aka Location). This child
# location needs to actually match one of the children of this
# Block. (expected invariant that we'll need to test, and handle
# authoring tools that mess this up)
# TODO: is there a way to add some validation around this, to
# be run on course load or in studio or ....
group_id_to_child = Dict(help="Which child module students in a particular "
"group_id should see",
scope=Scope.content)
@XBlock.needs('user_tags')
@XBlock.needs('partitions')
class SplitTestModule(SplitTestFields, XModule):
"""
Show the user the appropriate child. Uses the ExperimentState
API to figure out which child to show.
Course staff still get put in an experimental condition, but have the option
to see the other conditions. The only thing that counts toward their
grade/progress is the condition they are actually in.
Technical notes:
- There is more dark magic in this code than I'd like. The whole varying-children +
grading interaction is a tangle between super and subclasses of descriptors and
modules.
"""
def __init__(self, *args, **kwargs):
super(SplitTestModule, self).__init__(*args, **kwargs)
self.child_descriptor = self.get_child_descriptors()[0]
if self.child_descriptor is not None:
self.child = self.system.get_module(self.child_descriptor)
else:
self.child = None
def get_child_descriptor_by_location(self, location):
"""
Look through the children and look for one with the given location.
Returns the descriptor.
If none match, return None
"""
# NOTE: calling self.get_children() creates a circular reference--
# it calls get_child_descriptors() internally, but that doesn't work until
# we've picked a choice. Use self.descriptor.get_children() instead.
for child in self.descriptor.get_children():
if child.location.url() == location:
return child
return None
def get_child_descriptors(self):
"""
For grading--return just the chosen child.
"""
group_id = self.runtime.service(self, 'partitions').get_user_group_for_partition(self.user_partition_id)
# group_id_to_child comes from json, so it has to have string keys
str_group_id = str(group_id)
if str_group_id in self.group_id_to_child:
child_location = self.group_id_to_child[str_group_id]
child_descriptor = self.get_child_descriptor_by_location(child_location)
else:
# Oops. Config error.
log.debug("configuration error in split test module: invalid group_id %r (not one of %r). Showing error", str_group_id, self.group_id_to_child.keys())
if child_descriptor is None:
# Peak confusion is great. Now that we set child_descriptor,
# get_children() should return a list with one element--the
# xmodule for the child
log.debug("configuration error in split test module: no such child")
return []
return [child_descriptor]
def _staff_view(self, context):
"""
Render the staff view for a split test module.
"""
fragment = Fragment()
contents = []
for group_id in self.group_id_to_child:
child_location = self.group_id_to_child[group_id]
child_descriptor = self.get_child_descriptor_by_location(child_location)
child = self.system.get_module(child_descriptor)
rendered_child = child.render('student_view', context)
fragment.add_frag_resources(rendered_child)
contents.append({
'group_id': group_id,
'id': child.id,
'content': rendered_child.content
})
# Use the new template
fragment.add_content(self.system.render_template('split_test_staff_view.html', {
'items': contents,
}))
frag_js = """
$(document).ready(function() {{
ABTestSelector($('.split-test-view'));
}});
"""
fragment.add_javascript(frag_js)
fragment.add_javascript_url(self.runtime.local_resource_url(self, 'public/js/split_test_staff.js'))
return fragment
def student_view(self, context):
"""
Render the contents of the chosen condition for students, and all the
conditions for staff.
"""
if self.child is None:
# raise error instead? In fact, could complain on descriptor load...
return Fragment(content=u"<div>Nothing here. Move along.</div>")
if self.system.user_is_staff:
return self._staff_view(context)
else:
return self.child.render('student_view', context)
def get_icon_class(self):
return self.child.get_icon_class() if self.child else 'other'
def get_progress(self):
children = self.get_children()
progresses = [child.get_progress() for child in children]
progress = reduce(Progress.add_counts, progresses, None)
return progress
@XBlock.needs('user_tags')
@XBlock.needs('partitions')
class SplitTestDescriptor(SplitTestFields, SequenceDescriptor):
# the editing interface can be the same as for sequences -- just a container
module_class = SplitTestModule
filename_extension = "xml"
child_descriptor = module_attr('child_descriptor')
def definition_to_xml(self, resource_fs):
xml_object = etree.Element('split_test')
# TODO: also save the experiment id and the condition map
for child in self.get_children():
xml_object.append(
etree.fromstring(child.export_to_xml(resource_fs)))
return xml_object
def has_dynamic_children(self):
"""
Grading needs to know that only one of the children is actually "real". This
makes it use module.get_child_descriptors().
"""
return True

View File

@@ -0,0 +1,74 @@
"""
Tests for the Split Testing Module
"""
import ddt
from mock import Mock
from xmodule.tests.xml import factories as xml
from xmodule.tests.xml import XModuleXmlImportTest
from xmodule.tests import get_test_system
from xmodule.partitions.partitions import Group, UserPartition
from xmodule.partitions.partitions_service import PartitionService
from xmodule.partitions.test_partitions import StaticPartitionService
class SplitTestModuleFactory(xml.XmlImportFactory):
"""
Factory for generating SplitTestModules for testing purposes
"""
tag = 'split_test'
@ddt.ddt
class SplitTestModuleTest(XModuleXmlImportTest):
"""
Test the split test module
"""
def setUp(self):
self.course_id = 'test_org/test_course_number/test_run'
# construct module
course = xml.CourseFactory.build()
sequence = xml.SequenceFactory.build(parent=course)
split_test = SplitTestModuleFactory(
parent=sequence,
attribs={
'user_partition_id': '0',
'group_id_to_child': '{"0": "i4x://edX/xml_test_course/html/split_test_cond0", "1": "i4x://edX/xml_test_course/html/split_test_cond1"}'
}
)
xml.HtmlFactory(parent=split_test, url_name='split_test_cond0')
xml.HtmlFactory(parent=split_test, url_name='split_test_cond1')
self.course = self.process_xml(course)
course_seq = self.course.get_children()[0]
self.module_system = get_test_system()
self.tags_service = Mock(name='user_tags')
self.module_system._services['user_tags'] = self.tags_service
self.partitions_service = StaticPartitionService(
[
UserPartition(0, 'first_partition', 'First Partition', [Group("0", 'alpha'), Group("1", 'beta')]),
UserPartition(1, 'second_partition', 'Second Partition', [Group("0", 'abel'), Group("1", 'baker'), Group("2", 'charlie')])
],
user_tags_service=self.tags_service,
course_id=self.course.id,
track_function=Mock(name='track_function'),
)
self.module_system._services['partitions'] = self.partitions_service
self.split_test_descriptor = course_seq.get_children()[0]
self.split_test_descriptor.bind_for_student(
self.module_system,
self.split_test_descriptor._field_data
)
@ddt.data(('0', 'split_test_cond0'), ('1', 'split_test_cond1'))
@ddt.unpack
def test_child(self, user_tag, child_url_name):
self.tags_service.get_tag.return_value = user_tag
self.assertEquals(self.split_test_descriptor.child_descriptor.url_name, child_url_name)

View File

@@ -72,6 +72,7 @@ class XmlImportFactory(Factory):
url_name = Sequence(str)
attribs = {}
policy = {}
inline_xml = True
tag = 'unknown'
@classmethod
@@ -95,10 +96,26 @@ class XmlImportFactory(Factory):
kwargs['xml_node'].text = kwargs.pop('text', None)
kwargs['xml_node'].attrib.update(kwargs.pop('attribs', {}))
# Make sure that the xml_module doesn't try and open a file to find the contents
# of this node.
inline_xml = kwargs.pop('inline_xml')
if inline_xml:
kwargs['xml_node'].set('not_a_pointer', 'true')
for key in kwargs.keys():
if key not in XML_IMPORT_ARGS:
kwargs['xml_node'].set(key, kwargs.pop(key))
if not inline_xml:
kwargs['xml_node'].write(
kwargs['filesystem'].open(
'{}/{}.xml'.format(kwargs['tag'], kwargs['url_name'])
),
encoding='utf-8'
)
return kwargs
@lazy_attribute
@@ -134,3 +151,8 @@ class ProblemFactory(XmlImportFactory):
"""Factory for <problem> nodes"""
tag = 'problem'
text = '<h1>Empty Problem!</h1>'
class HtmlFactory(XmlImportFactory):
"""Factory for <problem> nodes"""
tag = 'html'