From 1823d9f09890b24943498e5d938881fdf4a72306 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Mon, 15 Jul 2019 10:48:34 -0400 Subject: [PATCH 1/2] Add django app for running Coverage who-tests-what in bokchoy --- cms/envs/bok_choy.py | 2 + cms/urls.py | 6 +++ lms/envs/bok_choy.py | 2 + lms/urls.py | 5 ++ openedx/testing/__init__.py | 0 .../coverage_context_listener/__init__.py | 0 .../testing/coverage_context_listener/apps.py | 9 ++++ .../pytest_plugin.py | 51 +++++++++++++++++++ .../testing/coverage_context_listener/urls.py | 9 ++++ .../coverage_context_listener/views.py | 24 +++++++++ pavelib/utils/test/suites/bokchoy_suite.py | 12 +++-- 11 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 openedx/testing/__init__.py create mode 100644 openedx/testing/coverage_context_listener/__init__.py create mode 100644 openedx/testing/coverage_context_listener/apps.py create mode 100644 openedx/testing/coverage_context_listener/pytest_plugin.py create mode 100644 openedx/testing/coverage_context_listener/urls.py create mode 100644 openedx/testing/coverage_context_listener/views.py diff --git a/cms/envs/bok_choy.py b/cms/envs/bok_choy.py index 7ed18f4fca..9e7a11e58e 100644 --- a/cms/envs/bok_choy.py +++ b/cms/envs/bok_choy.py @@ -172,6 +172,8 @@ VIDEO_TRANSCRIPTS_SETTINGS = dict( DIRECTORY_PREFIX='video-transcripts/', ) +INSTALLED_APPS.append('openedx.testing.coverage_context_listener') + ##################################################################### # Lastly, see if the developer has any local overrides. try: diff --git a/cms/urls.py b/cms/urls.py index 6feb0e506f..e6cb3a4d31 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -19,6 +19,7 @@ from openedx.core.djangoapps.password_policy import compliance as password_polic from openedx.core.djangoapps.password_policy.forms import PasswordPolicyAwareAdminAuthForm from openedx.core.openapi import schema_view + django_autodiscover() admin.site.site_header = _('Studio Administration') admin.site.site_title = admin.site.site_header @@ -275,5 +276,10 @@ if settings.FEATURES.get('ENABLE_API_DOCS'): url(r'^api-docs/$', schema_view.with_ui('swagger', cache_timeout=0)), ] +if 'openedx.testing.coverage_context_listener' in settings.INSTALLED_APPS: + urlpatterns += [ + url(r'coverage_context', include('openedx.testing.coverage_context_listener.urls')) + ] + from openedx.core.djangoapps.plugins import constants as plugin_constants, plugin_urls urlpatterns.extend(plugin_urls.get_patterns(plugin_constants.ProjectType.CMS)) diff --git a/lms/envs/bok_choy.py b/lms/envs/bok_choy.py index 2cf34aea64..374dcfa935 100644 --- a/lms/envs/bok_choy.py +++ b/lms/envs/bok_choy.py @@ -264,6 +264,8 @@ LMS_ROOT_URL = "http://localhost:{}".format(os.environ.get('BOK_CHOY_LMS_PORT', CMS_BASE = "localhost:{}".format(os.environ.get('BOK_CHOY_CMS_PORT', 8031)) LOGIN_REDIRECT_WHITELIST = [CMS_BASE] +INSTALLED_APPS.append('openedx.testing.coverage_context_listener') + if RELEASE_LINE == "master": # On master, acceptance tests use edX books, not the default Open edX books. HELP_TOKENS_BOOKS = { diff --git a/lms/urls.py b/lms/urls.py index c7f0a15dcd..39d04ff99c 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -976,4 +976,9 @@ urlpatterns += [ url(r'', include('csrf.urls')), ] +if 'openedx.testing.coverage_context_listener' in settings.INSTALLED_APPS: + urlpatterns += [ + url(r'coverage_context', include('openedx.testing.coverage_context_listener.urls')) + ] + urlpatterns.extend(plugin_urls.get_patterns(plugin_constants.ProjectType.LMS)) diff --git a/openedx/testing/__init__.py b/openedx/testing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openedx/testing/coverage_context_listener/__init__.py b/openedx/testing/coverage_context_listener/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openedx/testing/coverage_context_listener/apps.py b/openedx/testing/coverage_context_listener/apps.py new file mode 100644 index 0000000000..49ebd58269 --- /dev/null +++ b/openedx/testing/coverage_context_listener/apps.py @@ -0,0 +1,9 @@ +""" +Django AppConfig for the CoverageContextListener +""" +from django.apps import AppConfig + + +class CoverageContextListenerConfig(AppConfig): + name = 'openedx.testing.coverage_context_listener' + verbose_name = "Coverage Context Listener" diff --git a/openedx/testing/coverage_context_listener/pytest_plugin.py b/openedx/testing/coverage_context_listener/pytest_plugin.py new file mode 100644 index 0000000000..1e735014e7 --- /dev/null +++ b/openedx/testing/coverage_context_listener/pytest_plugin.py @@ -0,0 +1,51 @@ +""" +A pytest plugin that reports test contexts to coverage running in another process. +""" + +from pavelib.utils.envs import Env +import pytest +import requests + + +class ContextPlugin(object): + """ + Pytest plugin for reporting pytests contexts to coverage running in another process + """ + def __init__(self, config): + self.config = config + self.active = config.getoption("pytest-contexts") + + def pytest_runtest_setup(self, item): + self.doit(item, "setup") + + def pytest_runtest_teardown(self, item): + self.doit(item, "teardown") + + def pytest_runtest_call(self, item): + self.doit(item, "call") + + def doit(self, item, when): + if self.active: + for cfg in Env.BOK_CHOY_SERVERS.values(): + result = requests.post( + 'http://{host}:{port}/coverage_context/update_context'.format(**cfg), + { + 'context': "{}|{}".format(item.nodeid, when), + } + ) + assert result.status_code == 204 + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config): + config.pluginmanager.register(ContextPlugin(config), "contextplugin") + + +def pytest_addoption(parser): + group = parser.getgroup("coverage") + group.addoption( + "--pytest-contexts", + action="store_true", + dest="pytest-contexts", + help="Capture the pytest contexts that coverage is being captured in", + ) diff --git a/openedx/testing/coverage_context_listener/urls.py b/openedx/testing/coverage_context_listener/urls.py new file mode 100644 index 0000000000..6e0ce8b5f1 --- /dev/null +++ b/openedx/testing/coverage_context_listener/urls.py @@ -0,0 +1,9 @@ +""" +Coverage Context Listener URLs. +""" +from django.conf.urls import url +from .views import update_context + +urlpatterns = [ + url(r'update_context', update_context), +] diff --git a/openedx/testing/coverage_context_listener/views.py b/openedx/testing/coverage_context_listener/views.py new file mode 100644 index 0000000000..a139a4d8d8 --- /dev/null +++ b/openedx/testing/coverage_context_listener/views.py @@ -0,0 +1,24 @@ +""" +Views to allow modification of the current coverage context during test runs. +""" + +import coverage +from django.http import HttpResponse +from django.views.decorators.http import require_POST +from django.views.decorators.csrf import csrf_exempt + + +@require_POST +@csrf_exempt +def update_context(request): + """ + Set the current coverage context. + + POST data: + context: The current context + """ + context = request.POST.get('context') + current = coverage.Coverage.current() + if current is not None and context: + current.switch_context(context) + return HttpResponse(status=204) diff --git a/pavelib/utils/test/suites/bokchoy_suite.py b/pavelib/utils/test/suites/bokchoy_suite.py index 065920412a..e6d06de23e 100644 --- a/pavelib/utils/test/suites/bokchoy_suite.py +++ b/pavelib/utils/test/suites/bokchoy_suite.py @@ -204,7 +204,7 @@ class BokChoyTestSuite(TestSuite): check_services() if not self.testsonly: - call_task('prepare_bokchoy_run', options={'log_dir': self.log_dir}) + call_task('prepare_bokchoy_run', options={'log_dir': self.log_dir, 'coveragerc': self.coveragerc}) else: # load data in db_fixtures load_bok_choy_data() # pylint: disable=no-value-for-parameter @@ -322,8 +322,14 @@ class BokChoyTestSuite(TestSuite): cmd += [ "-m", "pytest", - test_spec, - ] + self.verbosity_processes_command + ] + if self.coveragerc: + cmd.extend([ + '-p', + 'openedx.testing.coverage_context_listener.pytest_plugin', + ]) + cmd.append(test_spec) + cmd.extend(self.verbosity_processes_command) if self.extra_args: cmd.append(self.extra_args) cmd.extend(self.passthrough_options) From 502556771139f25fd44bb23c1543f8c065e46bfe Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Tue, 16 Jul 2019 13:31:36 -0400 Subject: [PATCH 2/2] Add paver commands and environment variables to capture and run coverage-driven test selection --- pavelib/bok_choy.py | 7 +++- pavelib/tests.py | 19 +++++++++-- pavelib/utils/test/bokchoy_options.py | 6 ++++ pavelib/utils/test/suites/pytest_suite.py | 10 ++++++ pavelib/utils/test/utils.py | 39 +++++++++++++++++++++++ scripts/generic-ci-tests.sh | 6 ++++ scripts/jenkins-report.sh | 4 +++ scripts/unit-tests.sh | 7 ++++ 8 files changed, 94 insertions(+), 4 deletions(-) diff --git a/pavelib/bok_choy.py b/pavelib/bok_choy.py index f682cd5bb8..eacef701e2 100644 --- a/pavelib/bok_choy.py +++ b/pavelib/bok_choy.py @@ -6,7 +6,7 @@ from __future__ import absolute_import, print_function import os -from paver.easy import cmdopts, needs, sh, task +from paver.easy import cmdopts, needs, sh, task, call_task from pavelib.utils.envs import Env from pavelib.utils.passthrough_opts import PassthroughTask @@ -51,6 +51,11 @@ def test_bokchoy(options, passthrough_options): if validate_firefox: check_firefox_version() + if hasattr(options.test_bokchoy, 'with_wtw'): + call_task('fetch_coverage_test_selection_data', options={ + 'compare_branch': options.test_bokchoy.with_wtw + }) + run_bokchoy(options.test_bokchoy, passthrough_options) diff --git a/pavelib/tests.py b/pavelib/tests.py index 1d9ba50cfa..8598d2f2ba 100644 --- a/pavelib/tests.py +++ b/pavelib/tests.py @@ -8,7 +8,7 @@ import re import sys from optparse import make_option -from paver.easy import cmdopts, needs, sh, task +from paver.easy import cmdopts, needs, sh, task, call_task from pavelib.utils.envs import Env from pavelib.utils.passthrough_opts import PassthroughTask @@ -88,8 +88,16 @@ __test__ = False # do not collect '--xdist_ip_addresses', dest='xdist_ip_addresses', help="Comma separated string of ip addresses to shard tests to via xdist." - ) -], share_with=['pavelib.utils.test.utils.clean_reports_dir']) + ), + make_option( + '--with-wtw', + dest='with_wtw', + action='store', + help="Only run tests based on the lines changed relative to the specified branch" + ), +], share_with=[ + 'pavelib.utils.test.utils.clean_reports_dir', +]) @PassthroughTask @timed def test_system(options, passthrough_options): @@ -103,6 +111,11 @@ def test_system(options, passthrough_options): assert system in (None, 'lms', 'cms') assert django_version in (None, '1.8', '1.9', '1.10', '1.11') + if hasattr(options.test_system, 'with_wtw'): + call_task('fetch_coverage_test_selection_data', options={ + 'compare_branch': options.test_system.with_wtw + }) + if test_id: # Testing a single test ID. # Ensure the proper system for the test id. diff --git a/pavelib/utils/test/bokchoy_options.py b/pavelib/utils/test/bokchoy_options.py index d59fadada6..fd46cab74a 100644 --- a/pavelib/utils/test/bokchoy_options.py +++ b/pavelib/utils/test/bokchoy_options.py @@ -89,4 +89,10 @@ BOKCHOY_OPTS = [ dest="save_screenshots", help="deprecated in favor of save-screenshots" ), + make_option( + '--with-wtw', + dest='with_wtw', + action='store', + help="Only run tests based on the lines changed relative to the specified branch" + ), ] diff --git a/pavelib/utils/test/suites/pytest_suite.py b/pavelib/utils/test/suites/pytest_suite.py index 554de7b5ea..0f238abc9b 100644 --- a/pavelib/utils/test/suites/pytest_suite.py +++ b/pavelib/utils/test/suites/pytest_suite.py @@ -10,6 +10,7 @@ from glob import glob from pavelib.utils.envs import Env from pavelib.utils.test import utils as test_utils from pavelib.utils.test.suites.suite import TestSuite +from pavelib.utils.test.utils import COVERAGE_CACHE_BASELINE, COVERAGE_CACHE_BASEPATH, WHO_TESTS_WHAT_DIFF __test__ = False # do not collect @@ -47,6 +48,7 @@ class PytestSuite(TestSuite): self.xunit_report = self.report_dir / "nosetests.xml" self.cov_args = kwargs.get('cov_args', '') + self.with_wtw = kwargs.get('with_wtw', False) def __enter__(self): super(PytestSuite, self).__enter__() @@ -104,6 +106,14 @@ class PytestSuite(TestSuite): if self.fail_fast or env_fail_fast_set: opts.append("--exitfirst") + if self.with_wtw: + opts.extend([ + '--wtw', + '{}/{}'.format(COVERAGE_CACHE_BASEPATH, WHO_TESTS_WHAT_DIFF), + '--wtwdb', + '{}/{}'.format(COVERAGE_CACHE_BASEPATH, COVERAGE_CACHE_BASELINE) + ]) + return opts diff --git a/pavelib/utils/test/utils.py b/pavelib/utils/test/utils.py index 370dc59016..0ba1f6d429 100644 --- a/pavelib/utils/test/utils.py +++ b/pavelib/utils/test/utils.py @@ -11,6 +11,7 @@ from paver.easy import cmdopts, sh, task from pavelib.utils.envs import Env from pavelib.utils.timer import timed +from pavelib.utils.db_utils import get_file_from_s3, upload_to_s3 try: from bok_choy.browser import browser @@ -20,6 +21,12 @@ except ImportError: MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017')) MINIMUM_FIREFOX_VERSION = 28.0 +COVERAGE_CACHE_BUCKET = "edx-tools-coverage-caches" +COVERAGE_CACHE_BASEPATH = "test_root/who_tests_what" +COVERAGE_CACHE_BASELINE = "who_tests_what.{}.baseline".format(os.environ.get('TEST_SUITE', 'all')) +WHO_TESTS_WHAT_DIFF = "who_tests_what.diff" + + __test__ = False # do not collect @@ -150,3 +157,35 @@ def check_firefox_version(): debian_path=debian_path ) ) + + +@task +@cmdopts([ + ("compare-branch=", "b", "Branch to compare against, defaults to origin/master"), +]) +@timed +def fetch_coverage_test_selection_data(options): + """ + Set up the datafiles needed to run coverage-driven test selection (who-tests-what) + """ + + try: + os.makedirs(COVERAGE_CACHE_BASEPATH) + except OSError: + pass # Directory already exists + + sh(u'git diff $(git merge-base {} HEAD) > {}/{}'.format( + getattr(options, 'compare_branch', 'origin/master'), + COVERAGE_CACHE_BASEPATH, + WHO_TESTS_WHAT_DIFF + )) + get_file_from_s3(COVERAGE_CACHE_BUCKET, COVERAGE_CACHE_BASELINE, COVERAGE_CACHE_BASEPATH) + + +@task +def upload_coverage_to_s3(): + upload_to_s3( + COVERAGE_CACHE_BASELINE, + '{}/{}'.format(COVERAGE_CACHE_BASEPATH, 'reports/{}.coverage'.format(os.environ.get('TEST_SUITE'))), + COVERAGE_CACHE_BUCKET + ) diff --git a/scripts/generic-ci-tests.sh b/scripts/generic-ci-tests.sh index 90a9dc17b2..2acf3a5eaf 100755 --- a/scripts/generic-ci-tests.sh +++ b/scripts/generic-ci-tests.sh @@ -175,6 +175,12 @@ case "$TEST_SUITE" in "bok-choy") PAVER_ARGS="-n $NUMBER_OF_BOKCHOY_THREADS" + if [[ -n "$WHO_TESTS_WHAT" ]]; then + PAVER_ARGS="$PAVER_ARGS --with-wtw=origin/master" + fi + if [[ -n "$PYTEST_CONTEXTS" ]]; then + PAVER_ARGS="$PAVER_ARGS --pytest-contexts --coveragerc=common/test/acceptance/.coveragerc" + fi export BOKCHOY_HEADLESS=true case "$SHARD" in diff --git a/scripts/jenkins-report.sh b/scripts/jenkins-report.sh index c5476e6373..801920e9c1 100755 --- a/scripts/jenkins-report.sh +++ b/scripts/jenkins-report.sh @@ -27,3 +27,7 @@ fi # JUnit test reporter will fail the build # if it thinks test results are old touch `find . -name *.xml` || true + +if [[ -n "$PYTEST_CONTEXTS" ]]; then + paver upload_coverage_to_s3 +fi diff --git a/scripts/unit-tests.sh b/scripts/unit-tests.sh index c70ef2280e..add85ebb8d 100755 --- a/scripts/unit-tests.sh +++ b/scripts/unit-tests.sh @@ -50,6 +50,13 @@ else PARALLEL="--processes=-1" fi +if [[ -n "$WHO_TESTS_WHAT" ]]; then + PAVER_ARGS="$PAVER_ARGS --with-wtw" +fi +if [[ -n "$PYTEST_CONTEXTS" ]]; then + PAVER_ARGS="$PAVER_ARGS --pytest-contexts" +fi + case "${TEST_SUITE}" in "lms-unit")