Paver: Ability to run only servers or only tests

This would allow a user to set up and run servers, with an open prompt for killing
them. Likewise a user could open a different terminal session and run tests only.

How-to:
* At a terminal/ssh session, start bok-choy servers with
    `paver test_bokchoy --serversonly`
    (or, if you've already run collectstatic on your system:
    `paver test_bokchoy --serversonly --fasttest`)
* When the above is running, you can now open a separate terminal/ssh session
and run:
    `paver test_bokchoy -t my_tests --testsonly`

Keep in mind, the 'testsonly' flag does no setup. There is some minimal teardown; however,
such as clearing mongo and flushing the lms database. (Some tests have non-unique identifiers
and could not be run more than once.)

pavelib/bok_choy.py

@@ -22,6 +22,8 @@ __test__ = False  # do not collect
 @cmdopts([
     ('test_spec=', 't', 'Specific test to run'),
     ('fasttest', 'a', 'Skip some setup'),
+    ('serversonly', 'r', 'Prepare suite and leave servers running'),
+    ('testsonly', 'o', 'Assume servers are running and execute tests only'),
     ('extra_args=', 'e', 'adds as extra args to the test command'),
     ('default_store=', 's', 'Default modulestore'),
     make_option("--verbose", action="store_const", const=2, dest="verbosity"),
@@ -33,7 +35,10 @@ __test__ = False  # do not collect
 def test_bokchoy(options):
     """
     Run acceptance tests that use the bok-choy framework.
-    Skips some setup if `fasttest` is True.
+    Skips some static asset steps if `fasttest` is True.
+    Using 'serversonly' will prepare and run servers, leaving a process running in the terminal. At
+        the same time, a user can open a separate terminal and use 'testsonly' for executing tests against
+        those running servers.
 
     `test_spec` is a nose-style test specifier relative to the test directory
     Examples:
@@ -53,6 +58,8 @@ def test_bokchoy(options):
     opts = {
         'test_spec': getattr(options, 'test_spec', None),
         'fasttest': getattr(options, 'fasttest', False),
+        'serversonly': getattr(options, 'serversonly', False),
+        'testsonly': getattr(options, 'testsonly', False),
         'default_store': getattr(options, 'default_store', 'split'),
         'verbosity': getattr(options, 'verbosity', 2),
         'extra_args': getattr(options, 'extra_args', ''),

pavelib/paver_tests/test_paver_bok_choy_cmds.py

@@ -55,3 +55,7 @@ class TestPaverBokChoyCmd(unittest.TestCase):
         self.request.test_spec = "test_foo.py"
         self.request.default_store = "invalid"
         self.assertEqual(self.request.cmd.strip(), self._expected_command(self.request.test_spec, "invalid"))
+
+    def test_serversonly(self):
+        self.request.serversonly = True
+        self.assertEqual(self.request.cmd.strip(), "")

pavelib/utils/test/suites/bokchoy_suite.py

@@ -1,6 +1,8 @@
 """
 Class used for defining and running Bok Choy acceptance test suite
 """
+from time import sleep
+
 from paver.easy import sh
 from pavelib.utils.test.suites import TestSuite
 from pavelib.utils.envs import Env
@@ -23,7 +25,9 @@ class BokChoyTestSuite(TestSuite):
       log_dir - directory for test output
       report_dir - directory for reports (e.g., coverage) related to test execution
       xunit_report - directory for xunit-style output (xml)
-      fasttest - when set, skip various set-up tasks (e.g., DB migrations)
+      fasttest - when set, skip various set-up tasks (e.g., collectstatic)
+      serversonly - prepare and run the necessary servers, only stopping when interrupted with Ctrl-C
+      testsonly - assume servers are running (as per above) and run tests with no setup or cleaning of environment
       test_spec - when set, specifies test files, classes, cases, etc. See platform doc.
       default_store - modulestore to use when running tests (split or draft)
     """
@@ -35,6 +39,8 @@ class BokChoyTestSuite(TestSuite):
         self.xunit_report = self.report_dir / "xunit.xml"
         self.cache = Env.BOK_CHOY_CACHE
         self.fasttest = kwargs.get('fasttest', False)
+        self.serversonly = kwargs.get('serversonly', False)
+        self.testsonly = kwargs.get('testsonly', False)
         self.test_spec = kwargs.get('test_spec', None)
         self.default_store = kwargs.get('default_store', None)
         self.verbosity = kwargs.get('verbosity', 2)
@@ -55,9 +61,32 @@ class BokChoyTestSuite(TestSuite):
             test_utils.clean_test_files()
 
         msg = colorize('green', "Checking for mongo, memchache, and mysql...")
-        print(msg)
+        print msg
         bokchoy_utils.check_services()
 
+        if not self.testsonly:
+            self.prepare_bokchoy_run()
+
+        msg = colorize('green', "Confirming servers have started...")
+        print msg
+        bokchoy_utils.wait_for_test_servers()
+        if self.serversonly:
+            self.run_servers_continuously()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        super(BokChoyTestSuite, self).__exit__(exc_type, exc_value, traceback)
+
+        msg = colorize('green', "Cleaning up databases...")
+        print msg
+
+        # Clean up data we created in the databases
+        sh("./manage.py lms --settings bok_choy flush --traceback --noinput")
+        bokchoy_utils.clear_mongo()
+
+    def prepare_bokchoy_run(self):
+        """
+        Sets up and starts servers for bok-choy run. This includes any stubbed servers.
+        """
         sh("{}/scripts/reset-test-db.sh".format(Env.REPO_ROOT))
 
         if not self.fasttest:
@@ -90,32 +119,41 @@ class BokChoyTestSuite(TestSuite):
             )
 
         # Ensure the test servers are available
-        msg = colorize('green', "Starting test servers...")
-        print(msg)
+        msg = colorize('green', "Confirming servers are running...")
+        print msg
         bokchoy_utils.start_servers(self.default_store)
 
-        msg = colorize('green', "Waiting for servers to start...")
-        print(msg)
-        bokchoy_utils.wait_for_test_servers()
+    def run_servers_continuously(self):
+        """
+        Infinite loop. Servers will continue to run in the current session unless interrupted.
+        """
+        print 'Bok-choy servers running. Press Ctrl-C to exit...\n'
+        print 'Note: pressing Ctrl-C multiple times can corrupt noseid files and system state. Just press it once.\n'
 
-    def __exit__(self, exc_type, exc_value, traceback):
-        super(BokChoyTestSuite, self).__exit__(exc_type, exc_value, traceback)
-
-        msg = colorize('green', "Cleaning up databases...")
-        print(msg)
-
-        # Clean up data we created in the databases
-        sh("./manage.py lms --settings bok_choy flush --traceback --noinput")
-        bokchoy_utils.clear_mongo()
+        while True:
+            try:
+                sleep(10000)
+            except KeyboardInterrupt:
+                print "Stopping bok-choy servers.\n"
+                break
 
     @property
     def cmd(self):
+        """
+        This method composes the nosetests command to send to the terminal. If nosetests aren't being run,
+         the command returns an empty string.
+        """
         # Default to running all tests if no specific test is specified
         if not self.test_spec:
             test_spec = self.test_dir
         else:
             test_spec = self.test_dir / self.test_spec
 
+        # Skip any additional commands (such as nosetests) if running in
+        # servers only mode
+        if self.serversonly:
+            return ""
+
         # Construct the nosetests command, specifying where to save
         # screenshots and XUnit XML reports
         cmd = [