Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #397 from MITx/feature/server_split

Browse Source 
Model replication to course databases
This commit is contained in:
Calen Pennington
2012-08-13 06:39:03 -07:00
parent a984fbb65f 4ac7862975
commit a3d7c4892d
17 changed files with 563 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
cms/envs/test.py
View File

@@ -55,6 +55,17 @@ DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': ENV_ROOT / "db" / "cms.db",
},
# The following are for testing purposes...
'edX/toy/2012_Fall': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': ENV_ROOT / "db" / "course1.db",
},
'edx/full/6.002_Spring_2012': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': ENV_ROOT / "db" / "course2.db",
}
}

205
common/djangoapps/student/models.py
View File

@@ -1,5 +1,30 @@
"""
WE'RE USING MIGRATIONS!
Models for Student Information
Replication Notes
In our live deployment, we intend to run in a scenario where there is a pool of
Portal servers that hold the canoncial user information and that user
information is replicated to slave Course server pools. Each Course has a set of
servers that serves only its content and has users that are relevant only to it.
We replicate the following tables into the Course DBs where the user is
enrolled. Only the Portal servers should ever write to these models.
* UserProfile
* CourseEnrollment
We do a partial replication of:
* User -- Askbot extends this and uses the extra fields, so we replicate only
the stuff that comes with basic django_auth and ignore the rest.)
There are a couple different scenarios:
1. There's an update of User or UserProfile -- replicate it to all Course DBs
that the user is enrolled in (found via CourseEnrollment).
2. There's a change in CourseEnrollment. We need to push copies of UserProfile,
CourseEnrollment, and the base fields in User
Migration Notes
If you make changes to this model, be sure to create an appropriate migration
file and check it in at the same time as your model changes. To do that,
@@ -10,16 +35,41 @@ file and check it in at the same time as your model changes. To do that,
"""
from datetime import datetime
import json
import logging
import uuid
from django.db import models
from django.conf import settings
from django.contrib.auth.models import User
from django.db import models
from django.db.models.signals import post_delete, post_save
from django.dispatch import receiver
from django_countries import CountryField
from xmodule.modulestore.django import modulestore
#from cache_toolbox import cache_model, cache_relation
log = logging.getLogger(__name__)
class UserProfile(models.Model):
"""This is where we store all the user demographic fields. We have a
separate table for this rather than extending the built-in Django auth_user.
Notes:
* Some fields are legacy ones from the first run of 6.002, from which
we imported many users.
* Fields like name and address are intentionally open ended, to account
for international variations. An unfortunate side-effect is that we
cannot efficiently sort on last names for instance.
Replication:
* Only the Portal servers should ever modify this information.
* All fields are replicated into relevant Course databases
Some of the fields are legacy ones that were captured during the initial
MITx fall prototype.
"""
class Meta:
db_table = "auth_userprofile"
@@ -203,3 +253,154 @@ def add_user_to_default_group(user, group):
utg.save()
utg.users.add(User.objects.get(username=user))
utg.save()
########################## REPLICATION SIGNALS #################################
@receiver(post_save, sender=User)
def replicate_user_save(sender, **kwargs):
user_obj = kwargs['instance']
return replicate_model(User.save, user_obj, user_obj.id)
@receiver(post_save, sender=CourseEnrollment)
def replicate_enrollment_save(sender, **kwargs):
"""This is called when a Student enrolls in a course. It has to do the
following:
1. Make sure the User is copied into the Course DB. It may already exist
(someone deleting and re-adding a course). This has to happen first or
the foreign key constraint breaks.
2. Replicate the CourseEnrollment.
3. Replicate the UserProfile.
"""
if not is_portal():
return
enrollment_obj = kwargs['instance']
log.debug("Replicating user because of new enrollment")
replicate_user(enrollment_obj.user, enrollment_obj.course_id)
log.debug("Replicating enrollment because of new enrollment")
replicate_model(CourseEnrollment.save, enrollment_obj, enrollment_obj.user_id)
log.debug("Replicating user profile because of new enrollment")
user_profile = UserProfile.objects.get(user_id=enrollment_obj.user_id)
replicate_model(UserProfile.save, user_profile, enrollment_obj.user_id)
@receiver(post_delete, sender=CourseEnrollment)
def replicate_enrollment_delete(sender, **kwargs):
enrollment_obj = kwargs['instance']
return replicate_model(CourseEnrollment.delete, enrollment_obj, enrollment_obj.user_id)
@receiver(post_save, sender=UserProfile)
def replicate_userprofile_save(sender, **kwargs):
"""We just updated the UserProfile (say an update to the name), so push that
change to all Course DBs that we're enrolled in."""
user_profile_obj = kwargs['instance']
return replicate_model(UserProfile.save, user_profile_obj, user_profile_obj.user_id)
######### Replication functions #########
USER_FIELDS_TO_COPY = ["id", "username", "first_name", "last_name", "email",
"password", "is_staff", "is_active", "is_superuser",
"last_login", "date_joined"]
def replicate_user(portal_user, course_db_name):
"""Replicate a User to the correct Course DB. This is more complicated than
it should be because Askbot extends the auth_user table and adds its own
fields. So we need to only push changes to the standard fields and leave
the rest alone so that Askbot changes at the Course DB level don't get
overridden.
"""
try:
# If the user exists in the Course DB, update the appropriate fields and
# save it back out to the Course DB.
course_user = User.objects.using(course_db_name).get(id=portal_user.id)
for field in USER_FIELDS_TO_COPY:
setattr(course_user, field, getattr(portal_user, field))
mark_handled(course_user)
log.debug("User {0} found in Course DB, replicating fields to {1}"
.format(course_user, course_db_name))
course_user.save(using=course_db_name) # Just being explicit.
except User.DoesNotExist:
# Otherwise, just make a straight copy to the Course DB.
mark_handled(portal_user)
log.debug("User {0} not found in Course DB, creating copy in {1}"
.format(portal_user, course_db_name))
portal_user.save(using=course_db_name)
def replicate_model(model_method, instance, user_id):
"""
model_method is the model action that we want replicated. For instance,
UserProfile.save
"""
if not should_replicate(instance):
return
mark_handled(instance)
course_db_names = db_names_to_replicate_to(user_id)
log.debug("Replicating {0} for user {1} to DBs: {2}"
.format(model_method, user_id, course_db_names))
for db_name in course_db_names:
model_method(instance, using=db_name)
######### Replication Helpers #########
def is_valid_course_id(course_id):
"""Right now, the only database that's not a course database is 'default'.
I had nicer checking in here originally -- it would scan the courses that
were in the system and only let you choose that. But it was annoying to run
tests with, since we don't have course data for some for our course test
databases. Hence the lazy version.
"""
return course_id != 'default'
def is_portal():
"""Are we in the portal pool? Only Portal servers are allowed to replicate
their changes. For now, only Portal servers see multiple DBs, so we use
that to decide."""
return len(settings.DATABASES) > 1
def db_names_to_replicate_to(user_id):
"""Return a list of DB names that this user_id is enrolled in."""
return [c.course_id
for c in CourseEnrollment.objects.filter(user_id=user_id)
if is_valid_course_id(c.course_id)]
def marked_handled(instance):
"""Have we marked this instance as being handled to avoid infinite loops
caused by saving models in post_save hooks for the same models?"""
return hasattr(instance, '_do_not_copy_to_course_db')
def mark_handled(instance):
"""You have to mark your instance with this function or else we'll go into
an infinite loop since we're putting listeners on Model saves/deletes and
the act of replication requires us to call the same model method.
We create a _replicated attribute to differentiate the first save of this
model vs. the duplicate save we force on to the course database. Kind of
a hack -- suggestions welcome.
"""
instance._do_not_copy_to_course_db = True
def should_replicate(instance):
"""Should this instance be replicated? We need to be a Portal server and
the instance has to not have been marked_handled."""
if marked_handled(instance):
# Basically, avoid an infinite loop. You should
log.debug("{0} should not be replicated because it's been marked"
.format(instance))
return False
if not is_portal():
log.debug("{0} should not be replicated because we're not a portal."
.format(instance))
return False
return True

177
common/djangoapps/student/tests.py
View File

@@ -4,13 +4,178 @@ when you run "manage.py test".
Replace this with more appropriate tests for your application.
"""
from datetime import datetime
from django.test import TestCase
from .models import User, UserProfile, CourseEnrollment, replicate_user, USER_FIELDS_TO_COPY
COURSE_1 = 'edX/toy/2012_Fall'
COURSE_2 = 'edx/full/6.002_Spring_2012'
class ReplicationTest(TestCase):
multi_db = True
def test_user_replication(self):
"""Test basic user replication."""
portal_user = User.objects.create_user('rusty', 'rusty@edx.org', 'fakepass')
portal_user.first_name='Rusty'
portal_user.last_name='Skids'
portal_user.is_staff=True
portal_user.is_active=True
portal_user.is_superuser=True
portal_user.last_login=datetime(2012, 1, 1)
portal_user.date_joined=datetime(2011, 1, 1)
# This is an Askbot field and will break if askbot is not included
if hasattr(portal_user, 'seen_response_count'):
portal_user.seen_response_count = 10
portal_user.save(using='default')
# We replicate this user to Course 1, then pull the same user and verify
# that the fields copied over properly.
replicate_user(portal_user, COURSE_1)
course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
# Make sure the fields we care about got copied over for this user.
for field in USER_FIELDS_TO_COPY:
self.assertEqual(getattr(portal_user, field),
getattr(course_user, field),
"{0} not copied from {1} to {2}".format(
field, portal_user, course_user
))
if hasattr(portal_user, 'seen_response_count'):
# Since it's the first copy over of User data, we should have all of it
self.assertEqual(portal_user.seen_response_count,
course_user.seen_response_count)
# But if we replicate again, the user already exists in the Course DB,
# so it shouldn't update the seen_response_count (which is Askbot
# controlled).
# This hasattr lameness is here because we don't want this test to be
# triggered when we're being run by CMS tests (Askbot doesn't exist
# there, so the test will fail).
if hasattr(portal_user, 'seen_response_count'):
portal_user.seen_response_count = 20
replicate_user(portal_user, COURSE_1)
course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
self.assertEqual(portal_user.seen_response_count, 20)
self.assertEqual(course_user.seen_response_count, 10)
# Another replication should work for an email change however, since
# it's a field we care about.
portal_user.email = "clyde@edx.org"
replicate_user(portal_user, COURSE_1)
course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
self.assertEqual(portal_user.email, course_user.email)
# During this entire time, the user data should never have made it over
# to COURSE_2
self.assertRaises(User.DoesNotExist,
User.objects.using(COURSE_2).get,
id=portal_user.id)
def test_enrollment_for_existing_user_info(self):
"""Test the effect of Enrolling in a class if you've already got user
data to be copied over."""
# Create our User
portal_user = User.objects.create_user('jack', 'jack@edx.org', 'fakepass')
portal_user.first_name = "Jack"
portal_user.save()
# Set up our UserProfile info
portal_user_profile = UserProfile.objects.create(
user=portal_user,
name="Jack Foo",
level_of_education=None,
gender='m',
mailing_address=None,
goals="World domination",
)
portal_user_profile.save()
# Now let's see if creating a CourseEnrollment copies all the relevant
# data.
portal_enrollment = CourseEnrollment.objects.create(user=portal_user,
course_id=COURSE_1)
portal_enrollment.save()
# Grab all the copies we expect
course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
self.assertEquals(portal_user, course_user)
self.assertRaises(User.DoesNotExist,
User.objects.using(COURSE_2).get,
id=portal_user.id)
course_enrollment = CourseEnrollment.objects.using(COURSE_1).get(id=portal_enrollment.id)
self.assertEquals(portal_enrollment, course_enrollment)
self.assertRaises(CourseEnrollment.DoesNotExist,
CourseEnrollment.objects.using(COURSE_2).get,
id=portal_enrollment.id)
course_user_profile = UserProfile.objects.using(COURSE_1).get(id=portal_user_profile.id)
self.assertEquals(portal_user_profile, course_user_profile)
self.assertRaises(UserProfile.DoesNotExist,
UserProfile.objects.using(COURSE_2).get,
id=portal_user_profile.id)
def test_enrollment_for_user_info_after_enrollment(self):
"""Test the effect of modifying User data after you've enrolled."""
# Create our User
portal_user = User.objects.create_user('patty', 'patty@edx.org', 'fakepass')
portal_user.first_name = "Patty"
portal_user.save()
# Set up our UserProfile info
portal_user_profile = UserProfile.objects.create(
user=portal_user,
name="Patty Foo",
level_of_education=None,
gender='f',
mailing_address=None,
goals="World peace",
)
portal_user_profile.save()
# Now let's see if creating a CourseEnrollment copies all the relevant
# data when things are saved.
portal_enrollment = CourseEnrollment.objects.create(user=portal_user,
course_id=COURSE_1)
portal_enrollment.save()
portal_user.last_name = "Bar"
portal_user.save()
portal_user_profile.gender = 'm'
portal_user_profile.save()
# Grab all the copies we expect, and make sure it doesn't end up in
# places we don't expect.
course_user = User.objects.using(COURSE_1).get(id=portal_user.id)
self.assertEquals(portal_user, course_user)
self.assertRaises(User.DoesNotExist,
User.objects.using(COURSE_2).get,
id=portal_user.id)
course_enrollment = CourseEnrollment.objects.using(COURSE_1).get(id=portal_enrollment.id)
self.assertEquals(portal_enrollment, course_enrollment)
self.assertRaises(CourseEnrollment.DoesNotExist,
CourseEnrollment.objects.using(COURSE_2).get,
id=portal_enrollment.id)
course_user_profile = UserProfile.objects.using(COURSE_1).get(id=portal_user_profile.id)
self.assertEquals(portal_user_profile, course_user_profile)
self.assertRaises(UserProfile.DoesNotExist,
UserProfile.objects.using(COURSE_2).get,
id=portal_user_profile.id)
class SimpleTest(TestCase):
def test_basic_addition(self):
"""
Tests that 1 + 1 always equals 2.
"""
self.assertEqual(1 + 1, 2)

7
common/lib/xmodule/xmodule/modulestore/__init__.py
View File

@@ -190,6 +190,13 @@ class Location(_LocationBase):
return "Location%s" % repr(tuple(self))
@property
def course_id(self):
"""Return the ID of the Course that this item belongs to by looking
at the location URL hierachy"""
return "/".join([self.org, self.course, self.name])
class ModuleStore(object):
"""
An abstract interface for a database backend that stores XModuleDescriptor

24
lms/djangoapps/courseware/module_render.py
View File

@@ -2,6 +2,7 @@ import json
import logging
from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import Http404
from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt
@@ -148,12 +149,23 @@ def get_module(user, request, location, student_module_cache, position=None):
# TODO (vshnayder): fix hardcoded urls (use reverse)
# Setup system context for module instance
ajax_url = settings.MITX_ROOT_URL + '/modx/' + descriptor.location.url() + '/'
ajax_url = reverse('modx_dispatch',
kwargs=dict(course_id=descriptor.location.course_id,
id=descriptor.location.url(),
dispatch=''),
)
# ajax_url = settings.MITX_ROOT_URL + '/modx/' + descriptor.location.url() + '/'
# Fully qualified callback URL for external queueing system
xqueue_callback_url = (request.build_absolute_uri('/') + settings.MITX_ROOT_URL +
'xqueue/' + str(user.id) + '/' + descriptor.location.url() + '/' +
'score_update')
xqueue_callback_url = request.build_absolute_uri('/')[:-1] # Trailing slash provided by reverse
xqueue_callback_url += reverse('xqueue_callback',
kwargs=dict(course_id=descriptor.location.course_id,
userid=str(user.id),
id=descriptor.location.url(),
dispatch='score_update'),
)
# Default queuename is course-specific and is derived from the course that
# contains the current module.
@@ -259,7 +271,7 @@ def get_shared_instance_module(user, module, student_module_cache):
return None
@csrf_exempt
def xqueue_callback(request, userid, id, dispatch):
def xqueue_callback(request, course_id, userid, id, dispatch):
'''
Entry point for graded results from the queueing system.
'''
@@ -310,7 +322,7 @@ def xqueue_callback(request, userid, id, dispatch):
return HttpResponse("")
def modx_dispatch(request, dispatch=None, id=None):
def modx_dispatch(request, dispatch=None, id=None, course_id=None):
''' Generic view for extensions. This is where AJAX calls go.
Arguments:

0
lms/envs/devgroups/__init__.py Normal file
View File

43
lms/envs/devgroups/courses.py Normal file
View File

@@ -0,0 +1,43 @@
from ..dev import *
CLASSES_TO_DBS = {
'BerkeleyX/CS169.1x/2012_Fall' : "cs169.db",
'BerkeleyX/CS188.1x/2012_Fall' : "cs188_1.db",
'HarvardX/CS50x/2012' : "cs50.db",
'HarvardX/PH207x/2012_Fall' : "ph207.db",
'MITx/3.091x/2012_Fall' : "3091.db",
'MITx/6.002x/2012_Fall' : "6002.db",
'MITx/6.00x/2012_Fall' : "600.db",
}
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': '127.0.0.1:11211',
'KEY_FUNCTION': 'util.memcache.safe_key',
},
'general': {
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': '127.0.0.1:11211',
'KEY_PREFIX' : 'general',
'VERSION' : 5,
'KEY_FUNCTION': 'util.memcache.safe_key',
}
}
SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
def path_for_db(db_name):
return ENV_ROOT / "db" / db_name
def course_db_for(course_id):
db_name = CLASSES_TO_DBS[course_id]
return {
'default' : {
'ENGINE' : 'django.db.backends.sqlite3',
'NAME' : path_for_db(db_name)
}
}

3
lms/envs/devgroups/h_cs50.py Normal file
View File

@@ -0,0 +1,3 @@
from .courses import *
DATABASES = course_db_for('HarvardX/CS50x/2012')

3
lms/envs/devgroups/m_6002.py Normal file
View File

@@ -0,0 +1,3 @@
from .courses import *
DATABASES = course_db_for('MITx/6.002x/2012_Fall')

13
lms/envs/devgroups/portal.py Normal file
View File

@@ -0,0 +1,13 @@
"""
Note that for this to work at all, you must have memcached running (or you won't
get shared sessions)
"""
from courses import *
# Move this to a shared file later:
for class_id, db_name in CLASSES_TO_DBS.items():
DATABASES[class_id] = {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': path_for_db(db_name)
}

11
lms/envs/test.py
View File

@@ -67,6 +67,17 @@ DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': PROJECT_ROOT / "db" / "mitx.db",
},
# The following are for testing purposes...
'edX/toy/2012_Fall': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': ENV_ROOT / "db" / "course1.db",
},
'edx/full/6.002_Spring_2012': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': ENV_ROOT / "db" / "course2.db",
}
}

4
lms/templates/gradebook.html
View File

@@ -61,8 +61,8 @@
%for student in students:
<tr>
<td><a href="${reverse('student_profile',
kwargs={'course_id' : course_id,
'student_id': student['id']})}">
kwargs=dict(course_id=course_id,
student_id=student['id']))}">
${student['username']}</a></td>
%for section in student['grade_summary']['section_breakdown']:
${percent_data( section['percent'] )}

4
lms/templates/instructor_dashboard.html
View File

@@ -14,10 +14,10 @@
<h1>Instructor Dashboard</h1>
<p>
<a href="${reverse('gradebook', kwargs={'course_id': course.id})}">Gradebook</a>
<a href="${reverse('gradebook', kwargs=dict(course_id=course.id))}">Gradebook</a>
<p>
<a href="${reverse('grade_summary', kwargs={'course_id': course.id})}">Grade summary</a>
<a href="${reverse('grade_summary', kwargs=dict(course_id=course.id))}">Grade summary</a>
</section>
</div>

2
lms/templates/profile.html
View File

@@ -137,7 +137,7 @@ $(function() {
percentageString = "{0:.0%}".format( float(earned)/total) if earned > 0 and total > 0 else ""
%>
<h3><a href="${reverse('courseware_section', kwargs={'course_id' : course.id, 'chapter' : chapter['url_name'], 'section' : section['url_name']})}">
<h3><a href="${reverse('courseware_section', kwargs=dict(course_id=course.id, chapter=chapter['url_name'], section=section['url_name']))}">
${ section['display_name'] }</a><span> ${"({0:.3n}/{1:.3n}) {2}".format( float(earned), float(total), percentageString )}</span></h3>
<p>
${section['format']}

11
lms/urls.py
View File

@@ -2,7 +2,6 @@ from django.conf import settings
from django.conf.urls import patterns, include, url
from django.contrib import admin
from django.conf.urls.static import static
import django.contrib.auth.views
# Uncomment the next two lines to enable the admin:
@@ -101,10 +100,12 @@ if settings.COURSEWARE_ENABLED:
url(r'^masquerade/', include('masquerade.urls')),
url(r'^jump_to/(?P<location>.*)$', 'courseware.views.jump_to', name="jump_to"),
url(r'^modx/(?P<id>.*?)/(?P<dispatch>[^/]*)$',
'courseware.module_render.modx_dispatch', name='modx_dispatch'),
url(r'^xqueue/(?P<userid>[^/]*)/(?P<id>.*?)/(?P<dispatch>[^/]*)$',
'courseware.module_render.xqueue_callback', name='xqueue_callback'),
url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/modx/(?P<id>.*?)/(?P<dispatch>[^/]*)$',
'courseware.module_render.modx_dispatch',
name='modx_dispatch'),
url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/xqueue/(?P<userid>[^/]*)/(?P<id>.*?)/(?P<dispatch>[^/]*)$',
'courseware.module_render.xqueue_callback',
name='xqueue_callback'),
url(r'^change_setting$', 'student.views.change_setting',
name='change_setting'),

67
proxy/nginx.conf Normal file
View File

@@ -0,0 +1,67 @@
# Mapping of
#
# From the /mitx directory:
# /usr/local/Cellar/nginx/1.2.2/sbin/nginx -p `pwd`/ -c nginx.conf
worker_processes 1;
events {
worker_connections 1024;
}
http {
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
# server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /usr/local/etc/nginx/mime.types;
default_type application/octet-stream;
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
upstream portal {
server localhost:8000;
}
upstream course_harvardx_cs50_2012 {
server localhost:8001;
}
upstream course_mitx_6002_2012_fall {
server localhost:8002;
}
# Mostly copied from our existing server...
server {
listen 8100 default_server;
rewrite ^(.*)/favicon.ico$ /static/images/favicon.ico last;
# Our catchall
location / {
proxy_pass http://portal;
}
location /courses/HarvardX/CS50x/2012/ {
proxy_pass http://course_harvardx_cs50_2012;
}
location /courses/MITx/6.002x/2012_Fall/ {
proxy_pass http://course_mitx_6002_2012_fall;
}
}
}

3
rakefile
View File

@@ -88,7 +88,8 @@ $failed_tests = 0
def run_tests(system, report_dir, stop_on_failure=true)
ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
ENV['NOSE_COVER_HTML_DIR'] = File.join(report_dir, "cover")
sh(django_admin(system, :test, 'test', *Dir["#{system}/djangoapps/*"].each)) do |ok, res|
dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"]
sh(django_admin(system, :test, 'test', *dirs.each)) do |ok, res|
if !ok and stop_on_failure
abort "Test failed!"
end