From 468864a3146d93be0a9bbfb23873b6d9ed821099 Mon Sep 17 00:00:00 2001 From: Christine Lytwynec Date: Thu, 22 Oct 2015 13:35:40 -0400 Subject: [PATCH] add paver test_a11y and a11y_coverage commands --- common/test/acceptance/.a11ycoveragerc | 36 +++++++ common/test/acceptance/.coveragerc | 22 ++++- docs/en_us/internal/testing.rst | 16 +++- pavelib/bok_choy.py | 105 ++++++++++++++++----- pavelib/utils/envs.py | 2 + pavelib/utils/test/bokchoy_utils.py | 5 +- pavelib/utils/test/suites/bokchoy_suite.py | 5 +- scripts/accessibility-tests.sh | 5 +- 8 files changed, 165 insertions(+), 31 deletions(-) create mode 100644 common/test/acceptance/.a11ycoveragerc diff --git a/common/test/acceptance/.a11ycoveragerc b/common/test/acceptance/.a11ycoveragerc new file mode 100644 index 0000000000..6fab72971b --- /dev/null +++ b/common/test/acceptance/.a11ycoveragerc @@ -0,0 +1,36 @@ +[run] +data_file = reports/a11y/.coverage +source = + lms + cms + common/djangoapps + common/lib + openedx/core/djangoapps + +omit = + lms/envs/* + cms/envs/* + common/djangoapps/terrain/* + common/djangoapps/*/migrations/* + openedx/core/djangoapps/*/migrations/* + */test* + */management/* + */urls* + */wsgi* + lms/djangoapps/*/migrations/* + cms/djangoapps/*/migrations/* + +parallel = True + +[report] +ignore_errors = True +include = + **/views/*.py + **/views.py + +[html] +title = Bok Choy A11y Test Coverage Report +directory = reports/a11y/cover + +[xml] +output = reports/a11y/coverage.xml diff --git a/common/test/acceptance/.coveragerc b/common/test/acceptance/.coveragerc index 814241d2d9..e45ea2ece5 100644 --- a/common/test/acceptance/.coveragerc +++ b/common/test/acceptance/.coveragerc @@ -1,7 +1,25 @@ [run] data_file = reports/bok_choy/.coverage -source = lms, cms, common/djangoapps, common/lib -omit = lms/envs/*, cms/envs/*, common/djangoapps/terrain/*, common/djangoapps/*/migrations/*, openedx/core/djangoapps/*/migrations/*, */test*, */management/*, */urls*, */wsgi*, lms/djangoapps/*/migrations/*, cms/djangoapps/*/migrations/* +source = + lms + cms + common/djangoapps + common/lib + openedx/core/djangoapps + +omit = + lms/envs/* + cms/envs/* + common/djangoapps/terrain/* + common/djangoapps/*/migrations/* + openedx/core/djangoapps/*/migrations/* + */test* + */management/* + */urls* + */wsgi* + lms/djangoapps/*/migrations/* + cms/djangoapps/*/migrations/* + parallel = True [report] diff --git a/docs/en_us/internal/testing.rst b/docs/en_us/internal/testing.rst index f86c6b6249..4e34692c3b 100644 --- a/docs/en_us/internal/testing.rst +++ b/docs/en_us/internal/testing.rst @@ -456,14 +456,26 @@ To run all the bok choy accessibility tests use this command. :: - paver test_bokchoy --extra_args="-a 'a11y'" + paver test_a11y To run specific tests, use the ``-t`` flag to specify a nose-style test spec relative to the ``common/test/acceptance/tests`` directory. This is an example for it. :: - paver test_bokchoy --extra_args="-a 'a11y'" -t test_lms_dashboard.py:LmsDashboardA11yTest.test_dashboard_course_listings_a11y + paver test_a11y -t test_lms_dashboard.py:LmsDashboardA11yTest.test_dashboard_course_listings_a11y + +**Coverage**: + +To generate the coverage report for the views run during accessibility tests:: + + paver a11y_coverage + +Note that this coverage report is just a guideline to find areas that +are missing tests. If the view isn't 'covered', there definitely +isn't a test for it. If it is 'covered', we are loading that page +during the tests but not necessarily calling ``page.a11y_audit.check_for_accessibility_errors`` on it. + Options for Faster Development Cycles in Bok-Choy Tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/pavelib/bok_choy.py b/pavelib/bok_choy.py index d81801ca53..f21f95b5bb 100644 --- a/pavelib/bok_choy.py +++ b/pavelib/bok_choy.py @@ -16,10 +16,7 @@ except ImportError: __test__ = False # do not collect - -@task -@needs('pavelib.prereqs.install_prereqs') -@cmdopts([ +BOKCHOY_OPTS = [ ('test_spec=', 't', 'Specific test to run'), ('fasttest', 'a', 'Skip some setup'), ('serversonly', 'r', 'Prepare suite and leave servers running'), @@ -33,7 +30,32 @@ __test__ = False # do not collect make_option("-v", "--verbosity", action="count", dest="verbosity"), make_option("--pdb", action="store_true", help="Drop into debugger on failures or errors"), make_option("--skip_firefox_version_validation", action='store_false', dest="validate_firefox_version") -]) +] + + +def parse_bokchoy_opts(options): + """ + Parses bok choy options. + + Returns: dict of options. + """ + return { + 'test_spec': getattr(options, 'test_spec', None), + 'fasttest': getattr(options, 'fasttest', False), + 'num_processes': int(getattr(options, 'num_processes', 1)), + 'serversonly': getattr(options, 'serversonly', False), + 'testsonly': getattr(options, 'testsonly', False), + 'default_store': getattr(options, 'default_store', os.environ.get('DEFAULT_STORE', 'split')), + 'verbosity': getattr(options, 'verbosity', 2), + 'extra_args': getattr(options, 'extra_args', ''), + 'pdb': getattr(options, 'pdb', False), + 'test_dir': getattr(options, 'test_dir', 'tests'), + } + + +@task +@needs('pavelib.prereqs.install_prereqs') +@cmdopts(BOKCHOY_OPTS) def test_bokchoy(options): """ Run acceptance tests that use the bok-choy framework. @@ -57,18 +79,33 @@ def test_bokchoy(options): if validate_firefox: check_firefox_version() - opts = { - 'test_spec': getattr(options, 'test_spec', None), - 'num_processes': int(getattr(options, 'num_processes', 1)), - 'fasttest': getattr(options, 'fasttest', False), - 'serversonly': getattr(options, 'serversonly', False), - 'testsonly': getattr(options, 'testsonly', False), - 'default_store': getattr(options, 'default_store', os.environ.get('DEFAULT_STORE', 'split')), - 'verbosity': getattr(options, 'verbosity', 2), - 'extra_args': getattr(options, 'extra_args', ''), - 'pdb': getattr(options, 'pdb', False), - 'test_dir': getattr(options, 'test_dir', 'tests'), - } + opts = parse_bokchoy_opts(options) + run_bokchoy(**opts) + + +@task +@needs('pavelib.prereqs.install_prereqs') +@cmdopts(BOKCHOY_OPTS) +def test_a11y(options): + """ + Run accessibility tests that use the bok-choy framework. + Skips some static asset steps if `fasttest` is True. + Using 'serversonly' will prepare and run servers, leaving a process running in the terminal. At + the same time, a user can open a separate terminal and use 'testsonly' for executing tests against + those running servers. + + `test_spec` is a nose-style test specifier relative to the test directory + Examples: + - path/to/test.py + - path/to/test.py:TestFoo + - path/to/test.py:TestFoo.test_bar + It can also be left blank to run all tests in the suite that are tagged + with `@attr("a11y")`. + """ + opts = parse_bokchoy_opts(options) + opts['report_dir'] = Env.BOK_CHOY_A11Y_REPORT_DIR + opts['coveragerc'] = Env.BOK_CHOY_A11Y_COVERAGERC + opts['extra_args'] = opts['extra_args'] + ' -a "a11y" ' run_bokchoy(**opts) @@ -113,13 +150,11 @@ def run_bokchoy(**opts): test_suite.run() -@task -def bokchoy_coverage(): +def parse_coverage(report_dir, coveragerc): """ - Generate coverage reports for bok-choy tests + Generate coverage reports for bok-choy or a11y tests """ - Env.BOK_CHOY_REPORT_DIR.makedirs_p() - coveragerc = Env.BOK_CHOY_COVERAGERC + report_dir.makedirs_p() msg = colorize('green', "Combining coverage reports") print msg @@ -132,3 +167,29 @@ def bokchoy_coverage(): sh("coverage html --rcfile={}".format(coveragerc)) sh("coverage xml --rcfile={}".format(coveragerc)) sh("coverage report --rcfile={}".format(coveragerc)) + + +@task +def bokchoy_coverage(): + """ + Generate coverage reports for bok-choy tests + """ + parse_coverage( + Env.BOK_CHOY_REPORT_DIR, + Env.BOK_CHOY_COVERAGERC + ) + + +@task +def a11y_coverage(): + """ + Generate coverage reports for a11y tests. Note that this coverage report + is just a guideline to find areas that are missing tests. If the view + isn't 'covered', there definitely isn't a test for it. If it is + 'covered', we are loading that page during the tests but not necessarily + calling ``page.a11y_audit.check_for_accessibility_errors`` on it. + """ + parse_coverage( + Env.BOK_CHOY_A11Y_REPORT_DIR, + Env.BOK_CHOY_A11Y_COVERAGERC + ) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index 2a95543cf7..ac5b5be565 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -29,7 +29,9 @@ class Env(object): BOK_CHOY_DIR = REPO_ROOT / "common" / "test" / "acceptance" BOK_CHOY_LOG_DIR = REPO_ROOT / "test_root" / "log" BOK_CHOY_REPORT_DIR = REPORT_DIR / "bok_choy" + BOK_CHOY_A11Y_REPORT_DIR = REPORT_DIR / "a11y" BOK_CHOY_COVERAGERC = BOK_CHOY_DIR / ".coveragerc" + BOK_CHOY_A11Y_COVERAGERC = BOK_CHOY_DIR / ".a11ycoveragerc" # If set, put reports for run in "unique" directories. # The main purpose of this is to ensure that the reports can be 'slurped' diff --git a/pavelib/utils/test/bokchoy_utils.py b/pavelib/utils/test/bokchoy_utils.py index 23edbdd204..4153e58a45 100644 --- a/pavelib/utils/test/bokchoy_utils.py +++ b/pavelib/utils/test/bokchoy_utils.py @@ -18,10 +18,11 @@ except ImportError: __test__ = False # do not collect -def start_servers(default_store): +def start_servers(default_store, coveragerc=None): """ Start the servers we will run tests on, returns PIDs for servers. """ + coveragerc = coveragerc or Env.BOK_CHOY_COVERAGERC def start_server(cmd, logfile, cwd=None): """ @@ -38,7 +39,7 @@ def start_servers(default_store): "manage {service} --settings bok_choy runserver " "{address} --traceback --noreload".format( default_store=default_store, - coveragerc=Env.BOK_CHOY_COVERAGERC, + coveragerc=coveragerc, service=service, address=address, ) diff --git a/pavelib/utils/test/suites/bokchoy_suite.py b/pavelib/utils/test/suites/bokchoy_suite.py index 10842ef35e..7e87b15c33 100644 --- a/pavelib/utils/test/suites/bokchoy_suite.py +++ b/pavelib/utils/test/suites/bokchoy_suite.py @@ -43,7 +43,7 @@ class BokChoyTestSuite(TestSuite): super(BokChoyTestSuite, self).__init__(*args, **kwargs) self.test_dir = Env.BOK_CHOY_DIR / kwargs.get('test_dir', 'tests') self.log_dir = Env.BOK_CHOY_LOG_DIR - self.report_dir = Env.BOK_CHOY_REPORT_DIR + self.report_dir = kwargs.get('report_dir', Env.BOK_CHOY_REPORT_DIR) self.xunit_report = self.report_dir / "xunit.xml" self.cache = Env.BOK_CHOY_CACHE self.fasttest = kwargs.get('fasttest', False) @@ -56,6 +56,7 @@ class BokChoyTestSuite(TestSuite): self.extra_args = kwargs.get('extra_args', '') self.har_dir = self.log_dir / 'hars' self.imports_dir = kwargs.get('imports_dir', None) + self.coveragerc = kwargs.get('coveragerc', None) def __enter__(self): super(BokChoyTestSuite, self).__enter__() @@ -165,7 +166,7 @@ class BokChoyTestSuite(TestSuite): # Ensure the test servers are available msg = colorize('green', "Confirming servers are running...") print msg - bokchoy_utils.start_servers(self.default_store) + bokchoy_utils.start_servers(self.default_store, self.coveragerc) def run_servers_continuously(self): """ diff --git a/scripts/accessibility-tests.sh b/scripts/accessibility-tests.sh index d34b9e3bbc..98d1b5b03e 100755 --- a/scripts/accessibility-tests.sh +++ b/scripts/accessibility-tests.sh @@ -3,4 +3,7 @@ echo "Setting up for accessibility tests..." source scripts/jenkins-common.sh echo "Running explicit accessibility tests..." -SELENIUM_BROWSER=phantomjs paver test_bokchoy --extra_args="-a 'a11y'" +SELENIUM_BROWSER=phantomjs paver test_a11y + +echo "Generating coverage report..." +paver a11y_coverage