Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Ability to run bok-choy in multiprocess mode.

Browse Source 
This includes:
* Ability to specify number of processes to run bok-choy tests in
* A forked nose commit to get the multiprocess plugin's logging to work
* A different plugin (xunitmp) must be used for pulling together xunit results

This works by:
* Starting the various servers that are needed for the acceptance test environment
* Running the tests themselves in multiprocess mode
This commit is contained in:
Ben Patterson
2015-10-09 08:17:15 -04:00
parent 8103378ba6
commit 7eb9cecb9b
6 changed files with 126 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pavelib/bok_choy.py
View File

@@ -27,6 +27,7 @@ __test__ = False # do not collect
('extra_args=', 'e', 'adds as extra args to the test command'),
('default_store=', 's', 'Default modulestore'),
('test_dir=', 'd', 'Directory for finding tests (relative to common/test/acceptance)'),
('num_processes=', 'n', 'Number of test threads (for multiprocessing)'),
make_option("--verbose", action="store_const", const=2, dest="verbosity"),
make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"),
make_option("-v", "--verbosity", action="count", dest="verbosity"),
@@ -58,6 +59,7 @@ def test_bokchoy(options):
opts = {
'test_spec': getattr(options, 'test_spec', None),
'num_processes': int(getattr(options, 'num_processes', 1)),
'fasttest': getattr(options, 'fasttest', False),
'serversonly': getattr(options, 'serversonly', False),
'testsonly': getattr(options, 'testsonly', False),

64
pavelib/paver_tests/test_paver_bok_choy_cmds.py
View File

@@ -4,6 +4,7 @@ Run just this test with: paver test_lib -t pavelib/paver_tests/test_paver_bok_ch
"""
import os
import unittest
from paver.easy import BuildFailure
from pavelib.utils.test.suites import BokChoyTestSuite
REPO_DIR = os.getcwd()
@@ -19,7 +20,7 @@ class TestPaverBokChoyCmd(unittest.TestCase):
Returns the command that is expected to be run for the given test spec
and store.
"""
shard = os.environ.get('SHARD')
expected_statement = (
"DEFAULT_STORE={default_store} "
"SCREENSHOT_DIR='{repo_dir}/test_root/log{shard_str}' "
@@ -32,11 +33,15 @@ class TestPaverBokChoyCmd(unittest.TestCase):
).format(
default_store=store,
repo_dir=REPO_DIR,
shard_str='/shard_' + shard if shard else '',
shard_str='/shard_' + self.shard if self.shard else '',
exp_text=name,
)
return expected_statement
def setUp(self):
super(TestPaverBokChoyCmd, self).setUp()
self.shard = os.environ.get('SHARD')
def test_default(self):
suite = BokChoyTestSuite('')
name = 'tests'
@@ -89,3 +94,58 @@ class TestPaverBokChoyCmd(unittest.TestCase):
suite.cmd,
self._expected_command(name=test_dir)
)
def test_verbosity_settings_1_process(self):
"""
Using 1 process means paver should ask for the traditional xunit plugin for plugin results
"""
expected_verbosity_string = (
"--with-xunit --xunit-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml --verbosity=2".format(
repo_dir=REPO_DIR,
shard_str='/shard_' + self.shard if self.shard else ''
)
)
suite = BokChoyTestSuite('', num_processes=1)
self.assertEqual(BokChoyTestSuite.verbosity_processes_string(suite), expected_verbosity_string)
def test_verbosity_settings_2_processes(self):
"""
Using multiple processes means specific xunit, coloring, and process-related settings should
be used.
"""
process_count = 2
expected_verbosity_string = (
"--with-xunitmp --xunitmp-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml"
" --processes={procs} --no-color --process-timeout=1200".format(
repo_dir=REPO_DIR,
shard_str='/shard_' + self.shard if self.shard else '',
procs=process_count
)
)
suite = BokChoyTestSuite('', num_processes=process_count)
self.assertEqual(BokChoyTestSuite.verbosity_processes_string(suite), expected_verbosity_string)
def test_verbosity_settings_3_processes(self):
"""
With the above test, validate that num_processes can be set to various values
"""
process_count = 3
expected_verbosity_string = (
"--with-xunitmp --xunitmp-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml"
" --processes={procs} --no-color --process-timeout=1200".format(
repo_dir=REPO_DIR,
shard_str='/shard_' + self.shard if self.shard else '',
procs=process_count
)
)
suite = BokChoyTestSuite('', num_processes=process_count)
self.assertEqual(BokChoyTestSuite.verbosity_processes_string(suite), expected_verbosity_string)
def test_invalid_verbosity_and_processes(self):
"""
If an invalid combination of verbosity and number of processors is passed in, a
BuildFailure should be raised
"""
suite = BokChoyTestSuite('', num_processes=2, verbosity=3)
with self.assertRaises(BuildFailure):
BokChoyTestSuite.verbosity_processes_string(suite)

55
pavelib/utils/test/suites/bokchoy_suite.py
View File

@@ -3,7 +3,9 @@ Class used for defining and running Bok Choy acceptance test suite
"""
from time import sleep
from paver.easy import sh
from common.test.acceptance.fixtures.course import CourseFixture, FixtureError
from paver.easy import sh, BuildFailure
from pavelib.utils.test.suites.suite import TestSuite
from pavelib.utils.envs import Env
from pavelib.utils.test import bokchoy_utils
@@ -16,6 +18,9 @@ except ImportError:
__test__ = False # do not collect
DEFAULT_NUM_PROCESSES = 1
DEFAULT_VERBOSITY = 2
class BokChoyTestSuite(TestSuite):
"""
@@ -30,6 +35,9 @@ class BokChoyTestSuite(TestSuite):
testsonly - assume servers are running (as per above) and run tests with no setup or cleaning of environment
test_spec - when set, specifies test files, classes, cases, etc. See platform doc.
default_store - modulestore to use when running tests (split or draft)
num_processes - number of processes or threads to use in tests. Recommendation is that this
is less than or equal to the number of available processors.
See nosetest documentation: http://nose.readthedocs.org/en/latest/usage.html
"""
def __init__(self, *args, **kwargs):
super(BokChoyTestSuite, self).__init__(*args, **kwargs)
@@ -43,7 +51,8 @@ class BokChoyTestSuite(TestSuite):
self.testsonly = kwargs.get('testsonly', False)
self.test_spec = kwargs.get('test_spec', None)
self.default_store = kwargs.get('default_store', None)
self.verbosity = kwargs.get('verbosity', 2)
self.verbosity = kwargs.get('verbosity', DEFAULT_VERBOSITY)
self.num_processes = kwargs.get('num_processes', DEFAULT_NUM_PROCESSES)
self.extra_args = kwargs.get('extra_args', '')
self.har_dir = self.log_dir / 'hars'
self.imports_dir = kwargs.get('imports_dir', None)
@@ -70,6 +79,16 @@ class BokChoyTestSuite(TestSuite):
msg = colorize('green', "Confirming servers have started...")
print msg
bokchoy_utils.wait_for_test_servers()
try:
# Create course in order to seed forum data underneath. This is
# a workaround for a race condition. The first time a course is created;
# role permissions are set up for forums.
CourseFixture('foobar_org', '1117', 'seed_forum', 'seed_foo').install()
print 'Forums permissions/roles data has been seeded'
except FixtureError:
# this means it's already been done
pass
if self.serversonly:
self.run_servers_continuously()
@@ -83,6 +102,34 @@ class BokChoyTestSuite(TestSuite):
sh("./manage.py lms --settings bok_choy flush --traceback --noinput")
bokchoy_utils.clear_mongo()
def verbosity_processes_string(self):
"""
Multiprocessing, xunit, color, and verbosity do not work well together. We need to construct
the proper combination for use with nosetests.
"""
substring = []
if self.verbosity != DEFAULT_VERBOSITY and self.num_processes != DEFAULT_NUM_PROCESSES:
msg = 'Cannot pass in both num_processors and verbosity. Quitting'
raise BuildFailure(msg)
if self.num_processes != 1:
# Construct "multiprocess" nosetest substring
substring = [
"--with-xunitmp --xunitmp-file={}".format(self.xunit_report),
"--processes={}".format(self.num_processes),
"--no-color --process-timeout=1200"
]
else:
substring = [
"--with-xunit",
"--xunit-file={}".format(self.xunit_report),
"--verbosity={}".format(self.verbosity),
]
return " ".join(substring)
def prepare_bokchoy_run(self):
"""
Sets up and starts servers for a Bok Choy run. If --fasttest is not
@@ -160,9 +207,7 @@ class BokChoyTestSuite(TestSuite):
"SELENIUM_DRIVER_LOG_DIR='{}'".format(self.log_dir),
"nosetests",
test_spec,
"--with-xunit",
"--xunit-file={}".format(self.xunit_report),
"--verbosity={}".format(self.verbosity),
"{}".format(self.verbosity_processes_string())
]
if self.pdb:
cmd.append("--pdb")

2
requirements/edx/base.txt
View File

@@ -48,7 +48,7 @@ meliae==0.4.0
mongoengine==0.10.0
MySQL-python==1.2.5
networkx==1.7
nose==1.3.7
nose-xunitmp==0.3.2
oauthlib==0.7.2
paramiko==1.9.0
path.py==7.2

3
requirements/edx/github.txt
View File

@@ -33,6 +33,9 @@ git+https://github.com/edx/rfc6266.git@v0.0.5-edx#egg=rfc6266==0.0.5-edx
# Used for testing
-e git+https://github.com/gabrielfalcao/lettuce.git@b18b8fb711eb7a178c58574716032ad8de525912#egg=lettuce=1.8-support
# nose fork needed for multiprocess support
git+https://github.com/edx/nose.git@99c2aff0ff51bf228bfa5482e97e612c97a23245#egg=nose==1.3.7.1
# Our libraries:
-e git+https://github.com/edx/XBlock.git@a20c70f2e3df1cb716b9c7a25fecf57020543b7f#egg=XBlock
-e git+https://github.com/edx/codejail.git@6b17c33a89bef0ac510926b1d7fea2748b73aadd#egg=codejail

15
scripts/generic-ci-tests.sh
View File

@@ -54,6 +54,7 @@ set -e
# Note that you will still need to pass a value for 'TEST_SUITE'
# or else no tests will be executed.
SHARD=${SHARD:="all"}
NUMBER_OF_BOKCHOY_THREADS=${NUMBER_OF_BOKCHOY_THREADS:=1}
# Clean up previous builds
git clean -qxfd
@@ -148,31 +149,31 @@ END
;;
"1")
paver test_bokchoy --extra_args="-a shard_1 --with-flaky"
paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a shard_1 --with-flaky"
;;
"2")
paver test_bokchoy --extra_args="-a 'shard_2' --with-flaky"
paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_2' --with-flaky"
;;
"3")
paver test_bokchoy --extra_args="-a 'shard_3' --with-flaky"
paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_3' --with-flaky"
;;
"4")
paver test_bokchoy --extra_args="-a 'shard_4' --with-flaky"
paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_4' --with-flaky"
;;
"5")
paver test_bokchoy --extra_args="-a 'shard_5' --with-flaky"
paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_5' --with-flaky"
;;
"6")
paver test_bokchoy --extra_args="-a 'shard_6' --with-flaky"
paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_6' --with-flaky"
;;
"7")
paver test_bokchoy --extra_args="-a shard_1=False,shard_2=False,shard_3=False,shard_4=False,shard_5=False,shard_6=False,a11y=False --with-flaky"
paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a shard_1=False,shard_2=False,shard_3=False,shard_4=False,shard_5=False,shard_6=False,a11y=False --with-flaky"
;;
# Default case because if we later define another bok-choy shard on Jenkins