Merge pull request #21143 from cpennington/who-tests-what-nightly

Who tests what nightly

cms/envs/bok_choy.py

@@ -172,6 +172,8 @@ VIDEO_TRANSCRIPTS_SETTINGS = dict(
     DIRECTORY_PREFIX='video-transcripts/',
 )
 
+INSTALLED_APPS.append('openedx.testing.coverage_context_listener')
+
 #####################################################################
 # Lastly, see if the developer has any local overrides.
 try:

cms/urls.py

@@ -19,6 +19,7 @@ from openedx.core.djangoapps.password_policy import compliance as password_polic
 from openedx.core.djangoapps.password_policy.forms import PasswordPolicyAwareAdminAuthForm
 from openedx.core.openapi import schema_view
 
+
 django_autodiscover()
 admin.site.site_header = _('Studio Administration')
 admin.site.site_title = admin.site.site_header
@@ -275,5 +276,10 @@ if settings.FEATURES.get('ENABLE_API_DOCS'):
         url(r'^api-docs/$', schema_view.with_ui('swagger', cache_timeout=0)),
     ]
 
+if 'openedx.testing.coverage_context_listener' in settings.INSTALLED_APPS:
+    urlpatterns += [
+        url(r'coverage_context', include('openedx.testing.coverage_context_listener.urls'))
+    ]
+
 from openedx.core.djangoapps.plugins import constants as plugin_constants, plugin_urls
 urlpatterns.extend(plugin_urls.get_patterns(plugin_constants.ProjectType.CMS))

lms/envs/bok_choy.py

@@ -264,6 +264,8 @@ LMS_ROOT_URL = "http://localhost:{}".format(os.environ.get('BOK_CHOY_LMS_PORT',
 CMS_BASE = "localhost:{}".format(os.environ.get('BOK_CHOY_CMS_PORT', 8031))
 LOGIN_REDIRECT_WHITELIST = [CMS_BASE]
 
+INSTALLED_APPS.append('openedx.testing.coverage_context_listener')
+
 if RELEASE_LINE == "master":
     # On master, acceptance tests use edX books, not the default Open edX books.
     HELP_TOKENS_BOOKS = {

lms/urls.py

@@ -976,4 +976,9 @@ urlpatterns += [
     url(r'', include('csrf.urls')),
 ]
 
+if 'openedx.testing.coverage_context_listener' in settings.INSTALLED_APPS:
+    urlpatterns += [
+        url(r'coverage_context', include('openedx.testing.coverage_context_listener.urls'))
+    ]
+
 urlpatterns.extend(plugin_urls.get_patterns(plugin_constants.ProjectType.LMS))

openedx/testing/coverage_context_listener/apps.py

@@ -0,0 +1,9 @@
+"""
+Django AppConfig for the CoverageContextListener
+"""
+from django.apps import AppConfig
+
+
+class CoverageContextListenerConfig(AppConfig):
+    name = 'openedx.testing.coverage_context_listener'
+    verbose_name = "Coverage Context Listener"

openedx/testing/coverage_context_listener/pytest_plugin.py

@@ -0,0 +1,51 @@
+"""
+A pytest plugin that reports test contexts to coverage running in another process.
+"""
+
+from pavelib.utils.envs import Env
+import pytest
+import requests
+
+
+class ContextPlugin(object):
+    """
+    Pytest plugin for reporting pytests contexts to coverage running in another process
+    """
+    def __init__(self, config):
+        self.config = config
+        self.active = config.getoption("pytest-contexts")
+
+    def pytest_runtest_setup(self, item):
+        self.doit(item, "setup")
+
+    def pytest_runtest_teardown(self, item):
+        self.doit(item, "teardown")
+
+    def pytest_runtest_call(self, item):
+        self.doit(item, "call")
+
+    def doit(self, item, when):
+        if self.active:
+            for cfg in Env.BOK_CHOY_SERVERS.values():
+                result = requests.post(
+                    'http://{host}:{port}/coverage_context/update_context'.format(**cfg),
+                    {
+                        'context': "{}|{}".format(item.nodeid, when),
+                    }
+                )
+                assert result.status_code == 204
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_configure(config):
+    config.pluginmanager.register(ContextPlugin(config), "contextplugin")
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup("coverage")
+    group.addoption(
+        "--pytest-contexts",
+        action="store_true",
+        dest="pytest-contexts",
+        help="Capture the pytest contexts that coverage is being captured in",
+    )

openedx/testing/coverage_context_listener/urls.py

@@ -0,0 +1,9 @@
+"""
+Coverage Context Listener URLs.
+"""
+from django.conf.urls import url
+from .views import update_context
+
+urlpatterns = [
+    url(r'update_context', update_context),
+]

openedx/testing/coverage_context_listener/views.py

@@ -0,0 +1,24 @@
+"""
+Views to allow modification of the current coverage context during test runs.
+"""
+
+import coverage
+from django.http import HttpResponse
+from django.views.decorators.http import require_POST
+from django.views.decorators.csrf import csrf_exempt
+
+
+@require_POST
+@csrf_exempt
+def update_context(request):
+    """
+    Set the current coverage context.
+
+    POST data:
+        context: The current context
+    """
+    context = request.POST.get('context')
+    current = coverage.Coverage.current()
+    if current is not None and context:
+        current.switch_context(context)
+    return HttpResponse(status=204)

pavelib/bok_choy.py

@@ -6,7 +6,7 @@ from __future__ import absolute_import, print_function
 
 import os
 
-from paver.easy import cmdopts, needs, sh, task
+from paver.easy import cmdopts, needs, sh, task, call_task
 
 from pavelib.utils.envs import Env
 from pavelib.utils.passthrough_opts import PassthroughTask
@@ -51,6 +51,11 @@ def test_bokchoy(options, passthrough_options):
     if validate_firefox:
         check_firefox_version()
 
+    if hasattr(options.test_bokchoy, 'with_wtw'):
+        call_task('fetch_coverage_test_selection_data', options={
+            'compare_branch': options.test_bokchoy.with_wtw
+        })
+
     run_bokchoy(options.test_bokchoy, passthrough_options)
 
 

pavelib/tests.py

@@ -8,7 +8,7 @@ import re
 import sys
 from optparse import make_option
 
-from paver.easy import cmdopts, needs, sh, task
+from paver.easy import cmdopts, needs, sh, task, call_task
 
 from pavelib.utils.envs import Env
 from pavelib.utils.passthrough_opts import PassthroughTask
@@ -88,8 +88,16 @@ __test__ = False  # do not collect
         '--xdist_ip_addresses',
         dest='xdist_ip_addresses',
         help="Comma separated string of ip addresses to shard tests to via xdist."
-    )
-], share_with=['pavelib.utils.test.utils.clean_reports_dir'])
+    ),
+    make_option(
+        '--with-wtw',
+        dest='with_wtw',
+        action='store',
+        help="Only run tests based on the lines changed relative to the specified branch"
+    ),
+], share_with=[
+    'pavelib.utils.test.utils.clean_reports_dir',
+])
 @PassthroughTask
 @timed
 def test_system(options, passthrough_options):
@@ -103,6 +111,11 @@ def test_system(options, passthrough_options):
     assert system in (None, 'lms', 'cms')
     assert django_version in (None, '1.8', '1.9', '1.10', '1.11')
 
+    if hasattr(options.test_system, 'with_wtw'):
+        call_task('fetch_coverage_test_selection_data', options={
+            'compare_branch': options.test_system.with_wtw
+        })
+
     if test_id:
         # Testing a single test ID.
         # Ensure the proper system for the test id.

pavelib/utils/test/bokchoy_options.py

@@ -89,4 +89,10 @@ BOKCHOY_OPTS = [
         dest="save_screenshots",
         help="deprecated in favor of save-screenshots"
     ),
+    make_option(
+        '--with-wtw',
+        dest='with_wtw',
+        action='store',
+        help="Only run tests based on the lines changed relative to the specified branch"
+    ),
 ]

pavelib/utils/test/suites/bokchoy_suite.py

@@ -204,7 +204,7 @@ class BokChoyTestSuite(TestSuite):
         check_services()
 
         if not self.testsonly:
-            call_task('prepare_bokchoy_run', options={'log_dir': self.log_dir})
+            call_task('prepare_bokchoy_run', options={'log_dir': self.log_dir, 'coveragerc': self.coveragerc})
         else:
             # load data in db_fixtures
             load_bok_choy_data()  # pylint: disable=no-value-for-parameter
@@ -322,8 +322,14 @@ class BokChoyTestSuite(TestSuite):
         cmd += [
             "-m",
             "pytest",
-            test_spec,
-        ] + self.verbosity_processes_command
+        ]
+        if self.coveragerc:
+            cmd.extend([
+                '-p',
+                'openedx.testing.coverage_context_listener.pytest_plugin',
+            ])
+        cmd.append(test_spec)
+        cmd.extend(self.verbosity_processes_command)
         if self.extra_args:
             cmd.append(self.extra_args)
         cmd.extend(self.passthrough_options)

pavelib/utils/test/suites/pytest_suite.py

@@ -10,6 +10,7 @@ from glob import glob
 from pavelib.utils.envs import Env
 from pavelib.utils.test import utils as test_utils
 from pavelib.utils.test.suites.suite import TestSuite
+from pavelib.utils.test.utils import COVERAGE_CACHE_BASELINE, COVERAGE_CACHE_BASEPATH, WHO_TESTS_WHAT_DIFF
 
 __test__ = False  # do not collect
 
@@ -47,6 +48,7 @@ class PytestSuite(TestSuite):
         self.xunit_report = self.report_dir / "nosetests.xml"
 
         self.cov_args = kwargs.get('cov_args', '')
+        self.with_wtw = kwargs.get('with_wtw', False)
 
     def __enter__(self):
         super(PytestSuite, self).__enter__()
@@ -104,6 +106,14 @@ class PytestSuite(TestSuite):
         if self.fail_fast or env_fail_fast_set:
             opts.append("--exitfirst")
 
+        if self.with_wtw:
+            opts.extend([
+                '--wtw',
+                '{}/{}'.format(COVERAGE_CACHE_BASEPATH, WHO_TESTS_WHAT_DIFF),
+                '--wtwdb',
+                '{}/{}'.format(COVERAGE_CACHE_BASEPATH, COVERAGE_CACHE_BASELINE)
+            ])
+
         return opts
 
 

pavelib/utils/test/utils.py

@@ -11,6 +11,7 @@ from paver.easy import cmdopts, sh, task
 
 from pavelib.utils.envs import Env
 from pavelib.utils.timer import timed
+from pavelib.utils.db_utils import get_file_from_s3, upload_to_s3
 
 try:
     from bok_choy.browser import browser
@@ -20,6 +21,12 @@ except ImportError:
 MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017'))
 MINIMUM_FIREFOX_VERSION = 28.0
 
+COVERAGE_CACHE_BUCKET = "edx-tools-coverage-caches"
+COVERAGE_CACHE_BASEPATH = "test_root/who_tests_what"
+COVERAGE_CACHE_BASELINE = "who_tests_what.{}.baseline".format(os.environ.get('TEST_SUITE', 'all'))
+WHO_TESTS_WHAT_DIFF = "who_tests_what.diff"
+
+
 __test__ = False  # do not collect
 
 
@@ -150,3 +157,35 @@ def check_firefox_version():
                 debian_path=debian_path
             )
         )
+
+
+@task
+@cmdopts([
+    ("compare-branch=", "b", "Branch to compare against, defaults to origin/master"),
+])
+@timed
+def fetch_coverage_test_selection_data(options):
+    """
+    Set up the datafiles needed to run coverage-driven test selection (who-tests-what)
+    """
+
+    try:
+        os.makedirs(COVERAGE_CACHE_BASEPATH)
+    except OSError:
+        pass  # Directory already exists
+
+    sh(u'git diff $(git merge-base {} HEAD) > {}/{}'.format(
+        getattr(options, 'compare_branch', 'origin/master'),
+        COVERAGE_CACHE_BASEPATH,
+        WHO_TESTS_WHAT_DIFF
+    ))
+    get_file_from_s3(COVERAGE_CACHE_BUCKET, COVERAGE_CACHE_BASELINE, COVERAGE_CACHE_BASEPATH)
+
+
+@task
+def upload_coverage_to_s3():
+    upload_to_s3(
+        COVERAGE_CACHE_BASELINE,
+        '{}/{}'.format(COVERAGE_CACHE_BASEPATH, 'reports/{}.coverage'.format(os.environ.get('TEST_SUITE'))),
+        COVERAGE_CACHE_BUCKET
+    )

scripts/generic-ci-tests.sh

@@ -175,6 +175,12 @@ case "$TEST_SUITE" in
     "bok-choy")
 
         PAVER_ARGS="-n $NUMBER_OF_BOKCHOY_THREADS"
+        if [[ -n "$WHO_TESTS_WHAT" ]]; then
+            PAVER_ARGS="$PAVER_ARGS --with-wtw=origin/master"
+        fi
+        if [[ -n "$PYTEST_CONTEXTS" ]]; then
+            PAVER_ARGS="$PAVER_ARGS --pytest-contexts --coveragerc=common/test/acceptance/.coveragerc"
+        fi
         export BOKCHOY_HEADLESS=true
 
         case "$SHARD" in

scripts/jenkins-report.sh

@@ -27,3 +27,7 @@ fi
 # JUnit test reporter will fail the build
 # if it thinks test results are old
 touch `find . -name *.xml` || true
+
+if [[ -n "$PYTEST_CONTEXTS" ]]; then
+    paver upload_coverage_to_s3
+fi

scripts/unit-tests.sh

@@ -50,6 +50,13 @@ else
     PARALLEL="--processes=-1"
 fi
 
+if [[ -n "$WHO_TESTS_WHAT" ]]; then
+    PAVER_ARGS="$PAVER_ARGS --with-wtw"
+fi
+if [[ -n "$PYTEST_CONTEXTS" ]]; then
+    PAVER_ARGS="$PAVER_ARGS --pytest-contexts"
+fi
+
 case "${TEST_SUITE}" in
 
     "lms-unit")