diff --git a/pavelib/bok_choy.py b/pavelib/bok_choy.py index b76936fa40..75049b37bf 100644 --- a/pavelib/bok_choy.py +++ b/pavelib/bok_choy.py @@ -27,6 +27,7 @@ __test__ = False # do not collect ('extra_args=', 'e', 'adds as extra args to the test command'), ('default_store=', 's', 'Default modulestore'), ('test_dir=', 'd', 'Directory for finding tests (relative to common/test/acceptance)'), + ('num_processes=', 'n', 'Number of test threads (for multiprocessing)'), make_option("--verbose", action="store_const", const=2, dest="verbosity"), make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"), make_option("-v", "--verbosity", action="count", dest="verbosity"), @@ -58,6 +59,7 @@ def test_bokchoy(options): opts = { 'test_spec': getattr(options, 'test_spec', None), + 'num_processes': int(getattr(options, 'num_processes', 1)), 'fasttest': getattr(options, 'fasttest', False), 'serversonly': getattr(options, 'serversonly', False), 'testsonly': getattr(options, 'testsonly', False), diff --git a/pavelib/paver_tests/test_paver_bok_choy_cmds.py b/pavelib/paver_tests/test_paver_bok_choy_cmds.py index 1bfd71fba7..9df04d09e9 100644 --- a/pavelib/paver_tests/test_paver_bok_choy_cmds.py +++ b/pavelib/paver_tests/test_paver_bok_choy_cmds.py @@ -4,6 +4,7 @@ Run just this test with: paver test_lib -t pavelib/paver_tests/test_paver_bok_ch """ import os import unittest +from paver.easy import BuildFailure from pavelib.utils.test.suites import BokChoyTestSuite REPO_DIR = os.getcwd() @@ -19,7 +20,7 @@ class TestPaverBokChoyCmd(unittest.TestCase): Returns the command that is expected to be run for the given test spec and store. """ - shard = os.environ.get('SHARD') + expected_statement = ( "DEFAULT_STORE={default_store} " "SCREENSHOT_DIR='{repo_dir}/test_root/log{shard_str}' " @@ -32,11 +33,15 @@ class TestPaverBokChoyCmd(unittest.TestCase): ).format( default_store=store, repo_dir=REPO_DIR, - shard_str='/shard_' + shard if shard else '', + shard_str='/shard_' + self.shard if self.shard else '', exp_text=name, ) return expected_statement + def setUp(self): + super(TestPaverBokChoyCmd, self).setUp() + self.shard = os.environ.get('SHARD') + def test_default(self): suite = BokChoyTestSuite('') name = 'tests' @@ -89,3 +94,58 @@ class TestPaverBokChoyCmd(unittest.TestCase): suite.cmd, self._expected_command(name=test_dir) ) + + def test_verbosity_settings_1_process(self): + """ + Using 1 process means paver should ask for the traditional xunit plugin for plugin results + """ + expected_verbosity_string = ( + "--with-xunit --xunit-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml --verbosity=2".format( + repo_dir=REPO_DIR, + shard_str='/shard_' + self.shard if self.shard else '' + ) + ) + suite = BokChoyTestSuite('', num_processes=1) + self.assertEqual(BokChoyTestSuite.verbosity_processes_string(suite), expected_verbosity_string) + + def test_verbosity_settings_2_processes(self): + """ + Using multiple processes means specific xunit, coloring, and process-related settings should + be used. + """ + process_count = 2 + expected_verbosity_string = ( + "--with-xunitmp --xunitmp-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml" + " --processes={procs} --no-color --process-timeout=1200".format( + repo_dir=REPO_DIR, + shard_str='/shard_' + self.shard if self.shard else '', + procs=process_count + ) + ) + suite = BokChoyTestSuite('', num_processes=process_count) + self.assertEqual(BokChoyTestSuite.verbosity_processes_string(suite), expected_verbosity_string) + + def test_verbosity_settings_3_processes(self): + """ + With the above test, validate that num_processes can be set to various values + """ + process_count = 3 + expected_verbosity_string = ( + "--with-xunitmp --xunitmp-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml" + " --processes={procs} --no-color --process-timeout=1200".format( + repo_dir=REPO_DIR, + shard_str='/shard_' + self.shard if self.shard else '', + procs=process_count + ) + ) + suite = BokChoyTestSuite('', num_processes=process_count) + self.assertEqual(BokChoyTestSuite.verbosity_processes_string(suite), expected_verbosity_string) + + def test_invalid_verbosity_and_processes(self): + """ + If an invalid combination of verbosity and number of processors is passed in, a + BuildFailure should be raised + """ + suite = BokChoyTestSuite('', num_processes=2, verbosity=3) + with self.assertRaises(BuildFailure): + BokChoyTestSuite.verbosity_processes_string(suite) diff --git a/pavelib/utils/test/suites/bokchoy_suite.py b/pavelib/utils/test/suites/bokchoy_suite.py index 0328ad758a..10842ef35e 100644 --- a/pavelib/utils/test/suites/bokchoy_suite.py +++ b/pavelib/utils/test/suites/bokchoy_suite.py @@ -3,7 +3,9 @@ Class used for defining and running Bok Choy acceptance test suite """ from time import sleep -from paver.easy import sh +from common.test.acceptance.fixtures.course import CourseFixture, FixtureError + +from paver.easy import sh, BuildFailure from pavelib.utils.test.suites.suite import TestSuite from pavelib.utils.envs import Env from pavelib.utils.test import bokchoy_utils @@ -16,6 +18,9 @@ except ImportError: __test__ = False # do not collect +DEFAULT_NUM_PROCESSES = 1 +DEFAULT_VERBOSITY = 2 + class BokChoyTestSuite(TestSuite): """ @@ -30,6 +35,9 @@ class BokChoyTestSuite(TestSuite): testsonly - assume servers are running (as per above) and run tests with no setup or cleaning of environment test_spec - when set, specifies test files, classes, cases, etc. See platform doc. default_store - modulestore to use when running tests (split or draft) + num_processes - number of processes or threads to use in tests. Recommendation is that this + is less than or equal to the number of available processors. + See nosetest documentation: http://nose.readthedocs.org/en/latest/usage.html """ def __init__(self, *args, **kwargs): super(BokChoyTestSuite, self).__init__(*args, **kwargs) @@ -43,7 +51,8 @@ class BokChoyTestSuite(TestSuite): self.testsonly = kwargs.get('testsonly', False) self.test_spec = kwargs.get('test_spec', None) self.default_store = kwargs.get('default_store', None) - self.verbosity = kwargs.get('verbosity', 2) + self.verbosity = kwargs.get('verbosity', DEFAULT_VERBOSITY) + self.num_processes = kwargs.get('num_processes', DEFAULT_NUM_PROCESSES) self.extra_args = kwargs.get('extra_args', '') self.har_dir = self.log_dir / 'hars' self.imports_dir = kwargs.get('imports_dir', None) @@ -70,6 +79,16 @@ class BokChoyTestSuite(TestSuite): msg = colorize('green', "Confirming servers have started...") print msg bokchoy_utils.wait_for_test_servers() + try: + # Create course in order to seed forum data underneath. This is + # a workaround for a race condition. The first time a course is created; + # role permissions are set up for forums. + CourseFixture('foobar_org', '1117', 'seed_forum', 'seed_foo').install() + print 'Forums permissions/roles data has been seeded' + except FixtureError: + # this means it's already been done + pass + if self.serversonly: self.run_servers_continuously() @@ -83,6 +102,34 @@ class BokChoyTestSuite(TestSuite): sh("./manage.py lms --settings bok_choy flush --traceback --noinput") bokchoy_utils.clear_mongo() + def verbosity_processes_string(self): + """ + Multiprocessing, xunit, color, and verbosity do not work well together. We need to construct + the proper combination for use with nosetests. + """ + substring = [] + + if self.verbosity != DEFAULT_VERBOSITY and self.num_processes != DEFAULT_NUM_PROCESSES: + msg = 'Cannot pass in both num_processors and verbosity. Quitting' + raise BuildFailure(msg) + + if self.num_processes != 1: + # Construct "multiprocess" nosetest substring + substring = [ + "--with-xunitmp --xunitmp-file={}".format(self.xunit_report), + "--processes={}".format(self.num_processes), + "--no-color --process-timeout=1200" + ] + + else: + substring = [ + "--with-xunit", + "--xunit-file={}".format(self.xunit_report), + "--verbosity={}".format(self.verbosity), + ] + + return " ".join(substring) + def prepare_bokchoy_run(self): """ Sets up and starts servers for a Bok Choy run. If --fasttest is not @@ -160,9 +207,7 @@ class BokChoyTestSuite(TestSuite): "SELENIUM_DRIVER_LOG_DIR='{}'".format(self.log_dir), "nosetests", test_spec, - "--with-xunit", - "--xunit-file={}".format(self.xunit_report), - "--verbosity={}".format(self.verbosity), + "{}".format(self.verbosity_processes_string()) ] if self.pdb: cmd.append("--pdb") diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index bd0db987b3..a951e5fc42 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -48,7 +48,7 @@ meliae==0.4.0 mongoengine==0.10.0 MySQL-python==1.2.5 networkx==1.7 -nose==1.3.7 +nose-xunitmp==0.3.2 oauthlib==0.7.2 paramiko==1.9.0 path.py==7.2 diff --git a/requirements/edx/github.txt b/requirements/edx/github.txt index 478d1fb753..7171fdd10c 100644 --- a/requirements/edx/github.txt +++ b/requirements/edx/github.txt @@ -33,6 +33,9 @@ git+https://github.com/edx/rfc6266.git@v0.0.5-edx#egg=rfc6266==0.0.5-edx # Used for testing -e git+https://github.com/gabrielfalcao/lettuce.git@b18b8fb711eb7a178c58574716032ad8de525912#egg=lettuce=1.8-support +# nose fork needed for multiprocess support +git+https://github.com/edx/nose.git@99c2aff0ff51bf228bfa5482e97e612c97a23245#egg=nose==1.3.7.1 + # Our libraries: -e git+https://github.com/edx/XBlock.git@a20c70f2e3df1cb716b9c7a25fecf57020543b7f#egg=XBlock -e git+https://github.com/edx/codejail.git@6b17c33a89bef0ac510926b1d7fea2748b73aadd#egg=codejail diff --git a/scripts/generic-ci-tests.sh b/scripts/generic-ci-tests.sh index dd3208e837..32ea244271 100755 --- a/scripts/generic-ci-tests.sh +++ b/scripts/generic-ci-tests.sh @@ -54,6 +54,7 @@ set -e # Note that you will still need to pass a value for 'TEST_SUITE' # or else no tests will be executed. SHARD=${SHARD:="all"} +NUMBER_OF_BOKCHOY_THREADS=${NUMBER_OF_BOKCHOY_THREADS:=1} # Clean up previous builds git clean -qxfd @@ -148,31 +149,31 @@ END ;; "1") - paver test_bokchoy --extra_args="-a shard_1 --with-flaky" + paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a shard_1 --with-flaky" ;; "2") - paver test_bokchoy --extra_args="-a 'shard_2' --with-flaky" + paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_2' --with-flaky" ;; "3") - paver test_bokchoy --extra_args="-a 'shard_3' --with-flaky" + paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_3' --with-flaky" ;; "4") - paver test_bokchoy --extra_args="-a 'shard_4' --with-flaky" + paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_4' --with-flaky" ;; "5") - paver test_bokchoy --extra_args="-a 'shard_5' --with-flaky" + paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_5' --with-flaky" ;; "6") - paver test_bokchoy --extra_args="-a 'shard_6' --with-flaky" + paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a 'shard_6' --with-flaky" ;; "7") - paver test_bokchoy --extra_args="-a shard_1=False,shard_2=False,shard_3=False,shard_4=False,shard_5=False,shard_6=False,a11y=False --with-flaky" + paver test_bokchoy -n $NUMBER_OF_BOKCHOY_THREADS --extra_args="-a shard_1=False,shard_2=False,shard_3=False,shard_4=False,shard_5=False,shard_6=False,a11y=False --with-flaky" ;; # Default case because if we later define another bok-choy shard on Jenkins