PLAT-1676 Use pytest for bok-choy tests

2017-08-01 15:00:58 -04:00
parent 39cb750499
commit 9e3f35db22
11 changed files with 99 additions and 64 deletions
--- a/common/test/acceptance/.coveragerc
+++ b/common/test/acceptance/.coveragerc
@@ -10,21 +10,32 @@ source =
 omit =
    lms/envs/*
    cms/envs/*
+    cms/manage.py
+    cms/djangoapps/contentstore/views/dev.py
    common/djangoapps/terrain/*
    common/djangoapps/*/migrations/*
+    openedx/core/djangoapps/debug/*
    openedx/core/djangoapps/*/migrations/*
    */test*
    */management/*
    */urls*
    */wsgi*
+    lms/debug/*
+    lms/djangoapps/*/features/*
    lms/djangoapps/*/migrations/*
+    cms/djangoapps/*/features/*
    cms/djangoapps/*/migrations/*

+concurrency = multiprocessing
 parallel = True

 [report]
 ignore_errors = True

+exclude_lines =
+   pragma: no cover
+   raise NotImplementedError
+
 [html]
 title = Bok Choy Test Coverage Report
 directory = reports/bok_choy/cover
--- a/common/test/acceptance/tests/discussion/test_discussion.py
+++ b/common/test/acceptance/tests/discussion/test_discussion.py
@@ -1226,6 +1226,7 @@ class DiscussionUserProfileTest(UniqueCourseTest):
        self.profiled_user_id = self.setup_user(username=self.PROFILED_USERNAME)
        # now create a second user who will view the profile.
        self.user_id = self.setup_user()
+        UserProfileViewFixture([]).push()

    def setup_course(self):
        """
--- a/docs/testing.rst
+++ b/docs/testing.rst
@@ -437,14 +437,14 @@ To test only a certain feature, specify the file and the testcase class.

 ::

-    paver test_bokchoy -t studio/test_studio_bad_data.py:BadComponentTest
+    paver test_bokchoy -t studio/test_studio_bad_data.py::BadComponentTest

 To execute only a certain test case, specify the file name, class, and
 test case method.

 ::

-    paver test_bokchoy -t lms/test_lms.py:RegistrationTest.test_register
+    paver test_bokchoy -t lms/test_lms.py::RegistrationTest::test_register

 During acceptance test execution, log files and also screenshots of
 failed tests are captured in test\_root/log.
@@ -454,7 +454,7 @@ If you check this in, your tests will hang on jenkins.

 ::

-    from nose.tools import set_trace; set_trace()
+    import pdb; pdb.set_trace()

 By default, all bokchoy tests are run with the 'split' ModuleStore. To
 override the modulestore that is used, use the default\_store option.
@@ -506,7 +506,7 @@ relative to the ``common/test/acceptance/tests`` directory. This is an example f

 ::

-    paver test_a11y -t lms/test_lms_dashboard.py:LmsDashboardA11yTest.test_dashboard_course_listings_a11y
+    paver test_a11y -t lms/test_lms_dashboard.py::LmsDashboardA11yTest::test_dashboard_course_listings_a11y

 **Coverage**:

@@ -644,7 +644,7 @@ Running Tests on Paver Scripts

 To run tests on the scripts that power the various Paver commands, use the following command::

-  nosetests paver
+  nosetests pavelib


 Testing internationalization with dummy translations
--- a/pavelib/paver_tests/test_paver_bok_choy_cmds.py
+++ b/pavelib/paver_tests/test_paver_bok_choy_cmds.py
@@ -11,6 +11,7 @@ import ddt
 from mock import Mock, call, patch
 from paver.easy import BuildFailure, call_task, environment

+from pavelib.utils.envs import Env
 from pavelib.utils.test.suites import BokChoyTestSuite, Pa11yCrawler
 from pavelib.utils.test.suites.bokchoy_suite import DEMO_COURSE_IMPORT_DIR, DEMO_COURSE_TAR_GZ

@@ -40,10 +41,15 @@ class TestPaverBokChoyCmd(unittest.TestCase):
            ),
            "SELENIUM_DRIVER_LOG_DIR='{}/test_root/log{}'".format(REPO_DIR, shard_str),
            "VERIFY_XSS='{}'".format(verify_xss),
-            "nosetests",
+            "coverage",
+            "run",
+            "--rcfile={}".format(Env.BOK_CHOY_COVERAGERC),
+            "-m",
+            "pytest",
            "{}/common/test/acceptance/{}".format(REPO_DIR, name),
-            "--xunit-file={}/reports/bok_choy{}/xunit.xml".format(REPO_DIR, shard_str),
-            "--verbosity=2",
+            "--durations=20",
+            "--junitxml={}/reports/bok_choy{}/xunit.xml".format(REPO_DIR, shard_str),
+            "--verbose",
        ]
        return expected_statement

@@ -122,11 +128,11 @@ class TestPaverBokChoyCmd(unittest.TestCase):
        Using 1 process means paver should ask for the traditional xunit plugin for plugin results
        """
        expected_verbosity_command = [
-            "--xunit-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
+            "--junitxml={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
                repo_dir=REPO_DIR,
                shard_str='/shard_' + self.shard if self.shard else ''
            ),
-            "--verbosity=2",
+            "--verbose",
        ]
        suite = BokChoyTestSuite('', num_processes=1)
        self.assertEqual(suite.verbosity_processes_command, expected_verbosity_command)
@@ -138,13 +144,13 @@ class TestPaverBokChoyCmd(unittest.TestCase):
        """
        process_count = 2
        expected_verbosity_command = [
-            "--xunitmp-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
+            "--junitxml={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
                repo_dir=REPO_DIR,
                shard_str='/shard_' + self.shard if self.shard else '',
            ),
-            "--processes={}".format(process_count),
-            "--no-color",
-            "--process-timeout=1200",
+            "-n {}".format(process_count),
+            "--color=no",
+            "--verbose",
        ]
        suite = BokChoyTestSuite('', num_processes=process_count)
        self.assertEqual(suite.verbosity_processes_command, expected_verbosity_command)
@@ -155,27 +161,17 @@ class TestPaverBokChoyCmd(unittest.TestCase):
        """
        process_count = 3
        expected_verbosity_command = [
-            "--xunitmp-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
+            "--junitxml={repo_dir}/reports/bok_choy{shard_str}/xunit.xml".format(
                repo_dir=REPO_DIR,
                shard_str='/shard_' + self.shard if self.shard else '',
            ),
-            "--processes={}".format(process_count),
-            "--no-color",
-            "--process-timeout=1200",
+            "-n {}".format(process_count),
+            "--color=no",
+            "--verbose",
        ]
        suite = BokChoyTestSuite('', num_processes=process_count)
        self.assertEqual(suite.verbosity_processes_command, expected_verbosity_command)

-    def test_invalid_verbosity_and_processes(self):
-        """
-        If an invalid combination of verbosity and number of processors is passed in, a
-        BuildFailure should be raised
-        """
-        suite = BokChoyTestSuite('', num_processes=2, verbosity=3)
-        with self.assertRaises(BuildFailure):
-            # pylint: disable=pointless-statement
-            suite.verbosity_processes_command
-

@ddt.ddt
 class TestPaverPa11yCrawlerCmd(unittest.TestCase):
--- a/pavelib/paver_tests/test_utils.py
+++ b/pavelib/paver_tests/test_utils.py
@@ -6,9 +6,11 @@ import unittest

 from mock import patch

+from pavelib.utils.envs import Env
 from pavelib.utils.test.utils import MINIMUM_FIREFOX_VERSION, check_firefox_version


+@unittest.skipIf(Env.USING_DOCKER, 'Firefox version check works differently under Docker Devstack')
 class TestUtils(unittest.TestCase):
    """
    Test utils.py under pavelib/utils/test
--- a/pavelib/utils/test/bokchoy_options.py
+++ b/pavelib/utils/test/bokchoy_options.py
@@ -19,8 +19,16 @@ BOKCHOY_DEFAULT_STORE_DEPR = make_option(
    default=os.environ.get('DEFAULT_STORE', 'split'),
    help='deprecated in favor of default-store'
 )
-BOKCHOY_FASTTEST = make_option('-a', '--fasttest', action='store_true', help='Skip some setup')
-BOKCHOY_COVERAGERC = make_option('--coveragerc', help='coveragerc file to use during this test')
+BOKCHOY_EVAL_ATTR = make_option(
+    "-a", "--eval-attr",
+    dest="eval_attr", help="Only run tests matching given attribute expression."
+)
+BOKCHOY_FASTTEST = make_option('--fasttest', action='store_true', help='Skip some setup')
+BOKCHOY_COVERAGERC = make_option(
+    '--coveragerc',
+    default=Env.BOK_CHOY_COVERAGERC,
+    help='coveragerc file to use during this test'
+)

 BOKCHOY_OPTS = [
    ('test-spec=', 't', 'Specific test to run'),
@@ -28,7 +36,9 @@ BOKCHOY_OPTS = [
    ('skip-clean', 'C', 'Skip cleaning repository before running tests'),
    make_option('-r', '--serversonly', action='store_true', help='Prepare suite and leave servers running'),
    make_option('-o', '--testsonly', action='store_true', help='Assume servers are running and execute tests only'),
+    BOKCHOY_COVERAGERC,
    BOKCHOY_DEFAULT_STORE,
+    BOKCHOY_EVAL_ATTR,
    make_option(
        '-d', '--test-dir',
        default='tests',
--- a/pavelib/utils/test/suites/bokchoy_suite.py
+++ b/pavelib/utils/test/suites/bokchoy_suite.py
@@ -179,10 +179,11 @@ class BokChoyTestSuite(TestSuite):
      testsonly - assume servers are running (as per above) and run tests with no setup or cleaning of environment
      test_spec - when set, specifies test files, classes, cases, etc. See platform doc.
      default_store - modulestore to use when running tests (split or draft)
+      eval_attr - only run tests matching given attribute expression
      num_processes - number of processes or threads to use in tests. Recommendation is that this
      is less than or equal to the number of available processors.
      verify_xss - when set, check for XSS vulnerabilities in the page HTML.
-      See nosetest documentation: http://nose.readthedocs.org/en/latest/usage.html
+      See pytest documentation: https://docs.pytest.org/en/latest/
    """
    def __init__(self, *args, **kwargs):
        super(BokChoyTestSuite, self).__init__(*args, **kwargs)
@@ -196,6 +197,7 @@ class BokChoyTestSuite(TestSuite):
        self.testsonly = kwargs.get('testsonly', False)
        self.test_spec = kwargs.get('test_spec', None)
        self.default_store = kwargs.get('default_store', None)
+        self.eval_attr = kwargs.get('eval_attr', None)
        self.verbosity = kwargs.get('verbosity', DEFAULT_VERBOSITY)
        self.num_processes = kwargs.get('num_processes', DEFAULT_NUM_PROCESSES)
        self.verify_xss = kwargs.get('verify_xss', os.environ.get('VERIFY_XSS', True))
@@ -203,7 +205,7 @@ class BokChoyTestSuite(TestSuite):
        self.har_dir = self.log_dir / 'hars'
        self.a11y_file = Env.BOK_CHOY_A11Y_CUSTOM_RULES_FILE
        self.imports_dir = kwargs.get('imports_dir', None)
-        self.coveragerc = kwargs.get('coveragerc', None)
+        self.coveragerc = kwargs.get('coveragerc', Env.BOK_CHOY_COVERAGERC)
        self.save_screenshots = kwargs.get('save_screenshots', False)

    def __enter__(self):
@@ -269,29 +271,22 @@ class BokChoyTestSuite(TestSuite):
    @property
    def verbosity_processes_command(self):
        """
-        Multiprocessing, xunit, color, and verbosity do not work well together. We need to construct
-        the proper combination for use with nosetests.
+        Construct the proper combination of multiprocessing, XUnit XML file, color, and verbosity for use with pytest.
        """
-        command = []
-
-        if self.verbosity != DEFAULT_VERBOSITY and self.num_processes != DEFAULT_NUM_PROCESSES:
-            msg = 'Cannot pass in both num_processors and verbosity. Quitting'
-            raise BuildFailure(msg)
+        command = ["--junitxml={}".format(self.xunit_report)]

        if self.num_processes != 1:
-            # Construct "multiprocess" nosetest command
-            command = [
-                "--xunitmp-file={}".format(self.xunit_report),
-                "--processes={}".format(self.num_processes),
-                "--no-color",
-                "--process-timeout=1200",
-            ]
-
-        else:
-            command = [
-                "--xunit-file={}".format(self.xunit_report),
-                "--verbosity={}".format(self.verbosity),
+            # Construct "multiprocess" pytest command
+            command += [
+                "-n {}".format(self.num_processes),
+                "--color=no",
            ]
+        if self.verbosity < 1:
+            command.append("--quiet")
+        elif self.verbosity > 1:
+            command.append("--verbose")
+        if self.eval_attr:
+            command.append("-a '{}'".format(self.eval_attr))

        return command

@@ -300,7 +295,7 @@ class BokChoyTestSuite(TestSuite):
        Infinite loop. Servers will continue to run in the current session unless interrupted.
        """
        print 'Bok-choy servers running. Press Ctrl-C to exit...\n'
-        print 'Note: pressing Ctrl-C multiple times can corrupt noseid files and system state. Just press it once.\n'
+        print 'Note: pressing Ctrl-C multiple times can corrupt system state. Just press it once.\n'

        while True:
            try:
@@ -312,7 +307,7 @@ class BokChoyTestSuite(TestSuite):
    @property
    def cmd(self):
        """
-        This method composes the nosetests command to send to the terminal. If nosetests aren't being run,
+        This method composes the pytest command to send to the terminal. If pytest isn't being run,
         the command returns None.
        """
        # Default to running all tests if no specific test is specified
@@ -321,12 +316,12 @@ class BokChoyTestSuite(TestSuite):
        else:
            test_spec = self.test_dir / self.test_spec

-        # Skip any additional commands (such as nosetests) if running in
+        # Skip any additional commands (such as pytest) if running in
        # servers only mode
        if self.serversonly:
            return None

-        # Construct the nosetests command, specifying where to save
+        # Construct the pytest command, specifying where to save
        # screenshots and XUnit XML reports
        cmd = [
            "DEFAULT_STORE={}".format(self.default_store),
@@ -335,11 +330,21 @@ class BokChoyTestSuite(TestSuite):
            "BOKCHOY_A11Y_CUSTOM_RULES_FILE='{}'".format(self.a11y_file),
            "SELENIUM_DRIVER_LOG_DIR='{}'".format(self.log_dir),
            "VERIFY_XSS='{}'".format(self.verify_xss),
-            "nosetests",
-            test_spec,
-        ] + self.verbosity_processes_command
+        ]
        if self.save_screenshots:
-            cmd.append("--with-save-baseline")
+            cmd.append("NEEDLE_SAVE_BASELINE=True")
+        cmd += [
+            "coverage",
+            "run",
+        ]
+        if self.coveragerc:
+            cmd.append("--rcfile={}".format(self.coveragerc))
+        cmd += [
+            "-m",
+            "pytest",
+            test_spec,
+            "--durations=20",
+        ] + self.verbosity_processes_command
        if self.extra_args:
            cmd.append(self.extra_args)
        cmd.extend(self.passthrough_options)
--- a/requirements/edx/testing.txt
+++ b/requirements/edx/testing.txt
@@ -9,4 +9,11 @@
 #   * @edx/testeng - to discuss it's impact on test infrastructure
 #   * @edx/devops - to check system requirements

+execnet==1.4.1
+py==1.4.34
 pysqlite==2.8.3
+pytest==3.1.3
+pytest-attrib==0.1.3
+pytest-catchlog==1.2.2
+pytest-django==3.1.2
+pytest-xdist==1.18.1
--- a/scripts/accessibility-tests.sh
+++ b/scripts/accessibility-tests.sh
@@ -6,7 +6,7 @@ echo "Setting up for accessibility tests..."
 source scripts/jenkins-common.sh

 echo "Running explicit accessibility tests..."
-SELENIUM_BROWSER=phantomjs paver test_a11y --with-xunitmp
+SELENIUM_BROWSER=phantomjs paver test_a11y

 echo "Generating coverage report..."
 paver a11y_coverage
--- a/scripts/generic-ci-tests.sh
+++ b/scripts/generic-ci-tests.sh
@@ -162,7 +162,7 @@ case "$TEST_SUITE" in

    "bok-choy")

-        PAVER_ARGS="-n $NUMBER_OF_BOKCHOY_THREADS --with-flaky --with-xunit"
+        PAVER_ARGS="-n $NUMBER_OF_BOKCHOY_THREADS"

        case "$SHARD" in

@@ -171,11 +171,11 @@ case "$TEST_SUITE" in
                ;;

            [1-9]|10)
-                paver test_bokchoy --attr="shard=$SHARD" $PAVER_ARGS
+                paver test_bokchoy --eval-attr="shard==$SHARD" $PAVER_ARGS
                ;;

            11|"noshard")
-                paver test_bokchoy --attr='!shard,a11y=False' $PAVER_ARGS
+                paver test_bokchoy --eval-attr='not shard and not a11y' $PAVER_ARGS
                ;;

            # Default case because if we later define another bok-choy shard on Jenkins
@@ -190,7 +190,7 @@ case "$TEST_SUITE" in
                # May be unnecessary if we changed the "Skip if there are no test files"
                # option to True in the jenkins job definitions.
                mkdir -p reports/bok_choy
-                emptyxunit "bok_choy/nosetests"
+                emptyxunit "bok_choy/xunit"
                ;;
        esac
        ;;
--- a/setup.cfg
+++ b/setup.cfg
@@ -14,6 +14,9 @@ process-timeout=300
 #nocapture=1
 #pdb=1

+[tool:pytest]
+norecursedirs = .git conf node_modules test_root cms/envs lms/envs
+
 [pep8]
 # error codes: http://pep8.readthedocs.org/en/latest/intro.html#error-codes
 # E501: line too long