Merge pull request #6863 from edx/clytwynec/split_unittests_into_shards

split unittests into shards for in jenkins

pavelib/js_test.py

@@ -19,13 +19,15 @@ __test__ = False  # do not collect
     ("mode=", "m", "dev or run"),
     ("coverage", "c", "Run test under coverage"),
     ("port=", "p", "Port to run test server on (dev mode only)"),
-])
+    ('skip_clean', 'C', 'skip cleaning repository before running tests'),
+], share_with=["pavelib.utils.tests.utils.clean_reports_dir"])
 def test_js(options):
     """
     Run the JavaScript tests
     """
     mode = getattr(options, 'mode', 'run')
     port = None
+    skip_clean = getattr(options, 'skip_clean', False)
 
     if mode == 'run':
         suite = getattr(options, 'suite', 'all')
@@ -46,7 +48,7 @@ def test_js(options):
         )
         return
 
-    test_suite = JsTestSuite(suite, mode=mode, with_coverage=coverage, port=port)
+    test_suite = JsTestSuite(suite, mode=mode, with_coverage=coverage, port=port, skip_clean=skip_clean)
     test_suite.run()
 
 

pavelib/tests.py

@@ -3,7 +3,7 @@ Unit test tasks
 """
 import os
 import sys
-from paver.easy import sh, task, cmdopts, needs
+from paver.easy import sh, task, cmdopts, needs, call_task
 from pavelib.utils.test import suites
 from pavelib.utils.envs import Env
 from optparse import make_option
@@ -29,10 +29,11 @@ __test__ = False  # do not collect
     ("fasttest", "a", "Run without collectstatic"),
     ('extra_args=', 'e', 'adds as extra args to the test command'),
     ('cov_args=', 'c', 'adds as args to coverage for the test run'),
+    ('skip_clean', 'C', 'skip cleaning repository before running tests'),
     make_option("--verbose", action="store_const", const=2, dest="verbosity"),
     make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"),
     make_option("-v", "--verbosity", action="count", dest="verbosity", default=1),
-])
+], share_with=['pavelib.utils.test.utils.clean_reports_dir'])
 def test_system(options):
     """
     Run tests on our djangoapps for lms and cms
@@ -47,6 +48,7 @@ def test_system(options):
         'verbosity': getattr(options, 'verbosity', 1),
         'extra_args': getattr(options, 'extra_args', ''),
         'cov_args': getattr(options, 'cov_args', ''),
+        'skip_clean': getattr(options, 'skip_clean', False),
     }
 
     if test_id:
@@ -79,10 +81,11 @@ def test_system(options):
     ("fail_fast", "x", "Run only failed tests"),
     ('extra_args=', 'e', 'adds as extra args to the test command'),
     ('cov_args=', 'c', 'adds as args to coverage for the test run'),
+    ('skip_clean', 'C', 'skip cleaning repository before running tests'),
     make_option("--verbose", action="store_const", const=2, dest="verbosity"),
     make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"),
     make_option("-v", "--verbosity", action="count", dest="verbosity", default=1),
-])
+], share_with=['pavelib.utils.test.utils.clean_reports_dir'])
 def test_lib(options):
     """
     Run tests for common/lib/ and pavelib/ (paver-tests)
@@ -96,6 +99,7 @@ def test_lib(options):
         'verbosity': getattr(options, 'verbosity', 1),
         'extra_args': getattr(options, 'extra_args', ''),
         'cov_args': getattr(options, 'cov_args', ''),
+        'skip_clean': getattr(options, 'skip_clean', False),
     }
 
     if test_id:
@@ -151,6 +155,9 @@ def test_python(options):
     'pavelib.utils.test.utils.clean_reports_dir',
 )
 @cmdopts([
+    ("suites", "s", "List of unit test suites to run. (js, lib, cms, lms)"),
+    ('extra_args=', 'e', 'adds as extra args to the test command'),
+    ('cov_args=', 'c', 'adds as args to coverage for the test run'),
     make_option("--verbose", action="store_const", const=2, dest="verbosity"),
     make_option("-q", "--quiet", action="store_const", const=0, dest="verbosity"),
     make_option("-v", "--verbosity", action="count", dest="verbosity", default=1),
@@ -160,7 +167,9 @@ def test(options):
     Run all tests
     """
     opts = {
-        'verbosity': getattr(options, 'verbosity', 1)
+        'verbosity': getattr(options, 'verbosity', 1),
+        'extra_args': getattr(options, 'extra_args', ''),
+        'cov_args': getattr(options, 'cov_args', ''),
     }
     # Subsuites to be added to the main suite
     python_suite = suites.PythonTestSuite('Python Tests', **opts)
@@ -195,6 +204,21 @@ def coverage(options):
                 dir=directory
             ))
 
+
+    call_task('diff_coverage', options=dict(options))
+
+
+@task
+@needs('pavelib.prereqs.install_prereqs')
+@cmdopts([
+    ("compare_branch=", "b", "Branch to compare against, defaults to origin/master"),
+])
+def diff_coverage(options):
+    """
+    Build the diff coverage reports
+    """
+    compare_branch = getattr(options, 'compare_branch', 'origin/master')
+
     # Find all coverage XML files (both Python and JavaScript)
     xml_reports = []
 

pavelib/utils/test/suites/acceptance_suite.py

@@ -90,7 +90,8 @@ class AcceptanceTestSuite(TestSuite):
 
     def __enter__(self):
         super(AcceptanceTestSuite, self).__enter__()
-        test_utils.clean_test_files()
+        if not self.skip_clean:
+            test_utils.clean_test_files()
 
         if not self.fasttest:
             self._setup_acceptance_db()

pavelib/utils/test/suites/bokchoy_suite.py

@@ -51,7 +51,9 @@ class BokChoyTestSuite(TestSuite):
         self.har_dir.makedirs_p()
         self.report_dir.makedirs_p()
         test_utils.clean_reports_dir()
-        test_utils.clean_test_files()
+
+        if not self.skip_clean:
+            test_utils.clean_test_files()
 
         msg = colorize('green', "Checking for mongo, memchache, and mysql...")
         print(msg)

pavelib/utils/test/suites/js_suite.py

@@ -32,7 +32,8 @@ class JsTestSuite(TestSuite):
     def __enter__(self):
         super(JsTestSuite, self).__enter__()
         self.report_dir.makedirs_p()
-        test_utils.clean_test_files()
+        if not self.skip_clean:
+            test_utils.clean_test_files()
 
         if self.mode == 'run' and not self.run_under_coverage:
             test_utils.clean_dir(self.report_dir)

pavelib/utils/test/suites/python_suite.py

@@ -21,7 +21,7 @@ class PythonTestSuite(TestSuite):
 
     def __enter__(self):
         super(PythonTestSuite, self).__enter__()
-        if not self.fasttest:
+        if not (self.fasttest or self.skip_clean):
             test_utils.clean_test_files()
 
     @property

pavelib/utils/test/suites/suite.py

@@ -22,6 +22,7 @@ class TestSuite(object):
         self.subsuites = kwargs.get('subsuites', [])
         self.failed_suites = []
         self.verbosity = kwargs.get('verbosity', 1)
+        self.skip_clean = kwargs.get('skip_clean', False)
 
     def __enter__(self):
         """

pavelib/utils/test/utils.py

@@ -1,7 +1,7 @@
 """
 Helper functions for test tasks
 """
-from paver.easy import sh, task
+from paver.easy import sh, task, cmdopts
 from pavelib.utils.envs import Env
 import os
 import subprocess
@@ -33,10 +33,17 @@ def clean_dir(directory):
 
 
 @task
-def clean_reports_dir():
+@cmdopts([
+    ('skip_clean', 'C', 'skip cleaning repository before running tests'),
+])
+def clean_reports_dir(options):
     """
     Clean coverage files, to ensure that we don't use stale data to generate reports.
     """
+    if getattr(options, 'skip_clean', False):
+        print('--skip_clean is set, skipping...')
+        return
+
     # We delete the files but preserve the directory structure
     # so that coverage.py has a place to put the reports.
     reports_dir = Env.REPORT_DIR.makedirs_p()

scripts/all-tests.sh

@@ -55,47 +55,14 @@ set -e
 #
 ###############################################################################
 
-# Violations thresholds for failing the build
-PYLINT_THRESHOLD=6300
-PEP8_THRESHOLD=0
-
-source $HOME/jenkins_env
-
 # Clean up previous builds
 git clean -qxfd
 
-# Clear the mongo database
-# Note that this prevents us from running jobs in parallel on a single worker.
-mongo --quiet --eval 'db.getMongo().getDBNames().forEach(function(i){db.getSiblingDB(i).dropDatabase()})'
+source scripts/jenkins-common.sh
 
-# Ensure we have fetched origin/master
-# Some of the reporting tools compare the checked out branch to origin/master;
-# depending on how the GitHub plugin refspec is configured, this may
-# not already be fetched.
-git fetch origin master:refs/remotes/origin/master
-
-# Reset the jenkins worker's ruby environment back to
-# the state it was in when the instance was spun up.
-if [ -e $HOME/edx-rbenv_clean.tar.gz ]; then
-    rm -rf $HOME/.rbenv
-    tar -C $HOME -xf $HOME/edx-rbenv_clean.tar.gz
-fi
-
-# Bootstrap Ruby requirements so we can run the tests
-bundle install
-
-# Ensure the Ruby environment contains no stray gems
-bundle clean --force
-
-# Reset the jenkins worker's virtualenv back to the
-# state it was in when the instance was spun up.
-if [ -e $HOME/edx-venv_clean.tar.gz ]; then
-    rm -rf $HOME/edx-venv
-    tar -C $HOME -xf $HOME/edx-venv_clean.tar.gz
-fi
-
-# Activate the Python virtualenv
-source $HOME/edx-venv/bin/activate
+# Violations thresholds for failing the build
+PYLINT_THRESHOLD=6300
+PEP8_THRESHOLD=0
 
 # If the environment variable 'SHARD' is not set, default to 'all'.
 # This could happen if you are trying to use this script from
@@ -126,8 +93,22 @@ END
         ;;
 
     "unit")
-        paver test
-        paver coverage
+        case "$SHARD" in        
+            "lms")
+                paver test_system -s lms
+                paver coverage
+                ;;
+            "cms-js-commonlib")
+                paver test_system -s cms
+                paver test_js --coverage --skip_clean
+                paver test_lib --skip_clean
+                paver coverage
+                ;;
+            *)
+                paver test
+                paver coverage
+                ;;
+        esac
         ;;
 
     "lms-acceptance")

scripts/jenkins-common.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+source $HOME/jenkins_env
+
+# Clear the mongo database
+# Note that this prevents us from running jobs in parallel on a single worker.
+mongo --quiet --eval 'db.getMongo().getDBNames().forEach(function(i){db.getSiblingDB(i).dropDatabase()})'
+
+# Ensure we have fetched origin/master
+# Some of the reporting tools compare the checked out branch to origin/master;
+# depending on how the GitHub plugin refspec is configured, this may
+# not already be fetched.
+git fetch origin master:refs/remotes/origin/master
+
+# Reset the jenkins worker's ruby environment back to
+# the state it was in when the instance was spun up.
+if [ -e $HOME/edx-rbenv_clean.tar.gz ]; then
+    rm -rf $HOME/.rbenv
+    tar -C $HOME -xf $HOME/edx-rbenv_clean.tar.gz
+fi
+
+# Bootstrap Ruby requirements so we can run the tests
+bundle install
+
+# Ensure the Ruby environment contains no stray gems
+bundle clean --force
+
+# Reset the jenkins worker's virtualenv back to the
+# state it was in when the instance was spun up.
+if [ -e $HOME/edx-venv_clean.tar.gz ]; then
+    rm -rf $HOME/edx-venv
+    tar -C $HOME -xf $HOME/edx-venv_clean.tar.gz
+fi
+
+# Activate the Python virtualenv
+source $HOME/edx-venv/bin/activate

scripts/jenkins-report.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+source scripts/jenkins-common.sh
+
+# Run coverage again to get the diff coverage report
+paver diff_coverage
+
+# JUnit test reporter will fail the build
+# if it thinks test results are old
+touch `find . -name *.xml` || true