Deprecated unit tests from rake to paver

pavelib/utils/envs.py

@@ -20,6 +20,39 @@ class Env(object):
     # Reports Directory
     REPORT_DIR = REPO_ROOT / 'reports'
 
+    # Test Ids Directory
+    TEST_DIR = REPO_ROOT / ".testids"
+
+    # Files used to run each of the js test suites
+    # TODO:  Store this as a dict. Order seems to matter for some
+    # reason. See issue TE-415.
+    JS_TEST_ID_FILES = [
+        REPO_ROOT / 'lms/static/js_test.yml',
+        REPO_ROOT / 'cms/static/js_test.yml',
+        REPO_ROOT / 'cms/static/js_test_squire.yml',
+        REPO_ROOT / 'common/lib/xmodule/xmodule/js/js_test.yml',
+        REPO_ROOT / 'common/static/js_test.yml',
+    ]
+
+    JS_TEST_ID_KEYS = [
+        'lms',
+        'cms',
+        'cms-squire',
+        'xmodule',
+        'common',
+    ]
+
+    JS_REPORT_DIR = REPORT_DIR / 'javascript'
+
+    # Directories used for common/lib/ tests
+    LIB_TEST_DIRS = []
+    for item in (REPO_ROOT / "common/lib").listdir():
+        if (REPO_ROOT / 'common/lib' / item).isdir():
+            LIB_TEST_DIRS.append(path("common/lib") / item.basename())
+
+    # Directory for i18n test reports
+    I18N_REPORT_DIR = REPORT_DIR / 'i18n'
+
     # Service variant (lms, cms, etc.) configured with an environment variable
     # We use this to determine which envs.json file to load.
     SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None)

pavelib/utils/test/suites/__init__.py

@@ -0,0 +1,8 @@
+"""
+TestSuite class and subclasses
+"""
+from .suite import TestSuite
+from .nose_suite import NoseTestSuite, SystemTestSuite, LibTestSuite
+from .python_suite import PythonTestSuite
+from .js_suite import JsTestSuite
+from .i18n_suite import I18nTestSuite

pavelib/utils/test/suites/i18n_suite.py

@@ -0,0 +1,40 @@
+"""
+Classes used for defining and running i18n test suites
+"""
+from pavelib.utils.test.suites import TestSuite
+from pavelib.utils.envs import Env
+
+__test__ = False  # do not collect
+
+
+class I18nTestSuite(TestSuite):
+    """
+    Run tests for the internationalization library
+    """
+    def __init__(self, *args, **kwargs):
+        super(I18nTestSuite, self).__init__(*args, **kwargs)
+        self.report_dir = Env.I18N_REPORT_DIR
+        self.xunit_report = self.report_dir / 'nosetests.xml'
+
+    def __enter__(self):
+        super(I18nTestSuite, self).__enter__()
+        self.report_dir.makedirs_p()
+
+    @property
+    def cmd(self):
+        pythonpath_prefix = (
+            "PYTHONPATH={repo_root}/i18n:$PYTHONPATH".format(
+                repo_root=Env.REPO_ROOT
+            )
+        )
+
+        cmd = (
+            "{pythonpath_prefix} nosetests {repo_root}/i18n/tests "
+            "--with-xunit --xunit-file={xunit_report}".format(
+                pythonpath_prefix=pythonpath_prefix,
+                repo_root=Env.REPO_ROOT,
+                xunit_report=self.xunit_report,
+            )
+        )
+
+        return cmd

pavelib/utils/test/suites/js_suite.py

@@ -0,0 +1,61 @@
+"""
+Javascript test tasks
+"""
+from pavelib import assets
+from pavelib.utils.test import utils as test_utils
+from pavelib.utils.test.suites import TestSuite
+from pavelib.utils.envs import Env
+
+__test__ = False  # do not collect
+
+
+class JsTestSuite(TestSuite):
+    """
+    A class for running JavaScript tests.
+    """
+    def __init__(self, *args, **kwargs):
+        super(JsTestSuite, self).__init__(*args, **kwargs)
+        self.run_under_coverage = kwargs.get('with_coverage', True)
+        self.mode = kwargs.get('mode', 'run')
+
+        try:
+            self.test_id = (Env.JS_TEST_ID_FILES[Env.JS_TEST_ID_KEYS.index(self.root)])
+        except ValueError:
+            self.test_id = ' '.join(Env.JS_TEST_ID_FILES)
+
+        self.root = self.root + ' javascript'
+        self.report_dir = Env.JS_REPORT_DIR
+        self.coverage_report = self.report_dir / 'coverage.xml'
+        self.xunit_report = self.report_dir / 'javascript_xunit.xml'
+
+    def __enter__(self):
+        super(JsTestSuite, self).__enter__()
+        self.report_dir.makedirs_p()
+        test_utils.clean_test_files()
+
+        if self.mode == 'run' and not self.run_under_coverage:
+            test_utils.clean_dir(self.report_dir)
+
+        assets.compile_coffeescript("`find lms cms common -type f -name \"*.coffee\"`")
+
+    @property
+    def cmd(self):
+        """
+        Run the tests using js-test-tool. See js-test-tool docs for
+        description of different command line arguments.
+        """
+        cmd = (
+            "js-test-tool {mode} {test_id} --use-firefox --timeout-sec "
+            "600 --xunit-report {xunit_report}".format(
+                mode=self.mode,
+                test_id=self.test_id,
+                xunit_report=self.xunit_report,
+            )
+        )
+
+        if self.run_under_coverage:
+            cmd += " --coverage-xml {report_dir}".format(
+                report_dir=self.coverage_report
+            )
+
+        return cmd

pavelib/utils/test/suites/nose_suite.py

@@ -0,0 +1,169 @@
+"""
+Classes used for defining and running nose test suites
+"""
+import os
+from paver.easy import call_task
+from pavelib.utils.test import utils as test_utils
+from pavelib.utils.test.suites import TestSuite
+from pavelib.utils.envs import Env
+
+__test__ = False  # do not collect
+
+
+class NoseTestSuite(TestSuite):
+    """
+    A subclass of TestSuite with extra methods that are specific
+    to nose tests
+    """
+    def __init__(self, *args, **kwargs):
+        super(NoseTestSuite, self).__init__(*args, **kwargs)
+        self.failed_only = kwargs.get('failed_only', False)
+        self.fail_fast = kwargs.get('fail_fast', False)
+        self.run_under_coverage = kwargs.get('with_coverage', True)
+        self.report_dir = Env.REPORT_DIR / self.root
+        self.test_id_dir = Env.TEST_DIR / self.root
+        self.test_ids = self.test_id_dir / 'noseids'
+
+    def __enter__(self):
+        super(NoseTestSuite, self).__enter__()
+        self.report_dir.makedirs_p()
+        self.test_id_dir.makedirs_p()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        """
+        Cleans mongo afer the tests run.
+        """
+        super(NoseTestSuite, self).__exit__(exc_type, exc_value, traceback)
+        test_utils.clean_mongo()
+
+    def _under_coverage_cmd(self, cmd):
+        """
+        If self.run_under_coverage is True, it returns the arg 'cmd'
+        altered to be run under coverage. It returns the command
+        unaltered otherwise.
+        """
+        if self.run_under_coverage:
+            cmd0, cmd_rest = cmd.split(" ", 1)
+            # We use "python -m coverage" so that the proper python
+            # will run the importable coverage rather than the
+            # coverage that OS path finds.
+
+            cmd = (
+                "python -m coverage run --rcfile={root}/.coveragerc "
+                "`which {cmd0}` {cmd_rest}".format(
+                    root=self.root,
+                    cmd0=cmd0,
+                    cmd_rest=cmd_rest,
+                )
+            )
+
+        return cmd
+
+    @property
+    def test_options_flags(self):
+        """
+        Takes the test options and returns the appropriate flags
+        for the command.
+        """
+        opts = " "
+
+        # Handle "--failed" as a special case: we want to re-run only
+        # the tests that failed within our Django apps
+        # This sets the --failed flag for the nosetests command, so this
+        # functionality is the same as described in the nose documentation
+        if self.failed_only:
+            opts += "--failed"
+
+        # This makes it so we use nose's fail-fast feature in two cases.
+        # Case 1: --fail_fast is passed as an arg in the paver command
+        # Case 2: The environment variable TESTS_FAIL_FAST is set as True
+        env_fail_fast_set = (
+            'TESTS_FAIL_FAST' in os.environ and os.environ['TEST_FAIL_FAST']
+        )
+
+        if self.fail_fast or env_fail_fast_set:
+            opts += " --stop"
+
+        return opts
+
+
+class SystemTestSuite(NoseTestSuite):
+    """
+    TestSuite for lms and cms nosetests
+    """
+    def __init__(self, *args, **kwargs):
+        super(SystemTestSuite, self).__init__(*args, **kwargs)
+        self.test_id = kwargs.get('test_id', self._default_test_id)
+        self.fasttest = kwargs.get('fasttest', False)
+
+    def __enter__(self):
+        super(SystemTestSuite, self).__enter__()
+        args = [self.root, '--settings=test']
+
+        if self.fasttest:
+            # TODO: Fix the tests so that collectstatic isn't needed ever
+            # add --skip-collect to this when the tests are fixed
+            args.append('--skip-collect')
+
+        call_task('pavelib.assets.update_assets', args=args)
+
+    @property
+    def cmd(self):
+        cmd = (
+            './manage.py {system} test {test_id} {test_opts} '
+            '--traceback --settings=test'.format(
+                system=self.root,
+                test_id=self.test_id,
+                test_opts=self.test_options_flags,
+            )
+        )
+
+        return self._under_coverage_cmd(cmd)
+
+    @property
+    def _default_test_id(self):
+        """
+        If no test id is provided, we need to limit the test runner
+        to the Djangoapps we want to test.  Otherwise, it will
+        run tests on all installed packages. We do this by
+        using a default test id.
+        """
+        # We need to use $DIR/*, rather than just $DIR so that
+        # django-nose will import them early in the test process,
+        # thereby making sure that we load any django models that are
+        # only defined in test files.
+        default_test_id = "{system}/djangoapps/* common/djangoapps/*".format(
+            system=self.root
+        )
+
+        if self.root in ('lms', 'cms'):
+            default_test_id += " {system}/lib/*".format(system=self.root)
+
+        if self.root == 'lms':
+            default_test_id += " {system}/tests.py".format(system=self.root)
+
+        return default_test_id
+
+
+class LibTestSuite(NoseTestSuite):
+    """
+    TestSuite for edx-platform/common/lib nosetests
+    """
+    def __init__(self, *args, **kwargs):
+        super(LibTestSuite, self).__init__(*args, **kwargs)
+        self.test_id = kwargs.get('test_id', self.root)
+        self.xunit_report = self.report_dir / "nosetests.xml"
+
+    @property
+    def cmd(self):
+        cmd = (
+            "nosetests --id-file={test_ids} {test_id} {test_opts} "
+            "--with-xunit --xunit-file={xunit_report}".format(
+                test_ids=self.test_ids,
+                test_id=self.test_id,
+                test_opts=self.test_options_flags,
+                xunit_report=self.xunit_report,
+            )
+        )
+
+        return self._under_coverage_cmd(cmd)

pavelib/utils/test/suites/python_suite.py

@@ -0,0 +1,48 @@
+"""
+Classes used for defining and running python test suites
+"""
+from pavelib.utils.test import utils as test_utils
+from pavelib.utils.test.suites import TestSuite, LibTestSuite, SystemTestSuite
+from pavelib.utils.envs import Env
+
+__test__ = False  # do not collect
+
+
+class PythonTestSuite(TestSuite):
+    """
+    A subclass of TestSuite with extra setup for python tests
+    """
+    def __init__(self, *args, **kwargs):
+        super(PythonTestSuite, self).__init__(*args, **kwargs)
+        self.fasttest = kwargs.get('fasttest', False)
+        self.failed_only = kwargs.get('failed_only', None)
+        self.fail_fast = kwargs.get('fail_fast', None)
+        self.subsuites = kwargs.get('subsuites', self._default_subsuites)
+
+    def __enter__(self):
+        super(PythonTestSuite, self).__enter__()
+        if not self.fasttest:
+            test_utils.clean_test_files()
+
+    @property
+    def _default_subsuites(self):
+        """
+        The default subsuites to be run. They include lms, cms,
+        and all of the libraries in common/lib.
+        """
+        opts = {
+            'failed_only': self.failed_only,
+            'fail_fast': self.fail_fast,
+            'fasttest': self.fasttest,
+        }
+
+        lib_suites = [
+            LibTestSuite(d, **opts) for d in Env.LIB_TEST_DIRS
+        ]
+
+        system_suites = [
+            SystemTestSuite('cms', **opts),
+            SystemTestSuite('lms', **opts),
+        ]
+
+        return system_suites + lib_suites

pavelib/utils/test/suites/suite.py

@@ -0,0 +1,123 @@
+"""
+A class used for defining and running test suites
+"""
+import sys
+import subprocess
+from pavelib.utils.process import kill_process
+
+try:
+    from pygments.console import colorize
+except ImportError:
+    colorize = lambda color, text: text  # pylint: disable-msg=invalid-name
+
+__test__ = False  # do not collect
+
+
+class TestSuite(object):
+    """
+    TestSuite is a class that defines how groups of tests run.
+    """
+    def __init__(self, *args, **kwargs):
+        self.root = args[0]
+        self.subsuites = kwargs.get('subsuites', [])
+        self.failed_suites = []
+
+    def __enter__(self):
+        """
+        This will run before the test suite is run with the run_suite_tests method.
+        If self.run_test is called directly, it should be run in a 'with' block to
+        ensure that the proper context is created.
+
+        Specific setup tasks should be defined in each subsuite.
+
+        i.e. Checking for and defining required directories.
+        """
+        print("\nSetting up for {suite_name}".format(suite_name=self.root))
+        self.failed_suites = []
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        """
+        This is run after the tests run with the run_suite_tests method finish.
+        Specific clean up tasks should be defined in each subsuite.
+
+        If self.run_test is called directly, it should be run in a 'with' block
+        to ensure that clean up happens properly.
+
+        i.e. Cleaning mongo after the lms tests run.
+        """
+        print("\nCleaning up after {suite_name}".format(suite_name=self.root))
+
+    @property
+    def cmd(self):
+        """
+        The command to run tests (as a string). For this base class there is none.
+        """
+        return None
+
+    def run_test(self):
+        """
+        Runs a self.cmd in a subprocess and waits for it to finish.
+        It returns False if errors or failures occur. Otherwise, it
+        returns True.
+        """
+        cmd = self.cmd
+        sys.stdout.write(cmd)
+
+        msg = colorize(
+            'green',
+            '\n{bar}\n Running tests for {suite_name} \n{bar}\n'.format(suite_name=self.root, bar='=' * 40),
+        )
+
+        sys.stdout.write(msg)
+        sys.stdout.flush()
+
+        kwargs = {'shell': True, 'cwd': None}
+        process = None
+
+        try:
+            process = subprocess.Popen(cmd, **kwargs)
+            process.communicate()
+        except KeyboardInterrupt:
+            kill_process(process)
+            sys.exit(1)
+        else:
+            return (process.returncode == 0)
+
+    def run_suite_tests(self):
+        """
+        Runs each of the suites in self.subsuites while tracking failures
+        """
+        # Uses __enter__ and __exit__ for context
+        with self:
+            # run the tests for this class, and for all subsuites
+            if self.cmd:
+                passed = self.run_test()
+                if not passed:
+                    self.failed_suites.append(self)
+
+            for suite in self.subsuites:
+                suite.run_suite_tests()
+                if len(suite.failed_suites) > 0:
+                    self.failed_suites.extend(suite.failed_suites)
+
+    def report_test_results(self):
+        """
+        Writes a list of failed_suites to sys.stderr
+        """
+        if len(self.failed_suites) > 0:
+            msg = colorize('red', "\n\n{bar}\nTests failed in the following suites:\n* ".format(bar="=" * 48))
+            msg += colorize('red', '\n* '.join([s.root for s in self.failed_suites]) + '\n\n')
+        else:
+            msg = colorize('green', "\n\n{bar}\nNo test failures ".format(bar="=" * 48))
+
+        print(msg)
+
+    def run(self):
+        """
+        Runs the tests in the suite while tracking and reporting failures.
+        """
+        self.run_suite_tests()
+        self.report_test_results()
+
+        if len(self.failed_suites) > 0:
+            sys.exit(1)

pavelib/utils/test/utils.py

@@ -0,0 +1,45 @@
+"""
+Helper functions for test tasks
+"""
+from paver.easy import sh, task
+from pavelib.utils.envs import Env
+
+__test__ = False  # do not collect
+
+
+@task
+def clean_test_files():
+    """
+    Clean fixture files used by tests and .pyc files
+    """
+    sh("git clean -fqdx test_root/logs test_root/data test_root/staticfiles test_root/uploads")
+    sh("find . -type f -name \"*.pyc\" -delete")
+    sh("rm -rf test_root/log/auto_screenshots/*")
+
+
+def clean_dir(directory):
+    """
+    Clean coverage files, to ensure that we don't use stale data to generate reports.
+    """
+    # We delete the files but preserve the directory structure
+    # so that coverage.py has a place to put the reports.
+    sh('find {dir} -type f -delete'.format(dir=directory))
+
+
+@task
+def clean_reports_dir():
+    """
+    Clean coverage files, to ensure that we don't use stale data to generate reports.
+    """
+    # We delete the files but preserve the directory structure
+    # so that coverage.py has a place to put the reports.
+    reports_dir = Env.REPORT_DIR.makedirs_p()
+    clean_dir(reports_dir)
+
+
+@task
+def clean_mongo():
+    """
+    Clean mongo test databases
+    """
+    sh("mongo {repo_root}/scripts/delete-mongo-test-dbs.js".format(repo_root=Env.REPO_ROOT))