Merge pull request #21211 from cpennington/correct-combine

Check the test-suite specific coverage file, and upload it earlier in…

common/test/acceptance/.coveragerc

@@ -1,5 +1,5 @@
 [run]
-data_file = reports/bok_choy/.coverage
+data_file = reports/bok_choy/${TEST_SUITE}.coverage${SHARD}
 source =
     lms
     cms

pavelib/tests.py

@@ -322,21 +322,24 @@ def test(options, passthrough_options):
 @needs('pavelib.prereqs.install_coverage_prereqs')
 @cmdopts([
     ("compare-branch=", "b", "Branch to compare against, defaults to origin/master"),
+    ("rcfile=", "c", "Coveragerc file to use, defaults to .coveragerc"),
 ])
 @timed
-def coverage():
+def coverage(options):
     """
     Build the html, xml, and diff coverage reports
     """
     report_dir = Env.REPORT_DIR
-    rcfile = Env.PYTHON_COVERAGERC
+    rcfile = getattr(options.coverage, 'rcfile', Env.PYTHON_COVERAGERC)
 
-    if not (report_dir / '.coverage').isfile():
+    combined_report_file = report_dir / '{}.coverage'.format(os.environ.get('TEST_SUITE', ''))
+
+    if not combined_report_file.isfile():
         # This may be that the coverage files were generated using -p,
         # try to combine them to the one file that we need.
         sh(u"coverage combine --rcfile={}".format(rcfile))
 
-    if not os.path.getsize(report_dir / '.coverage') > 50:
+    if not os.path.getsize(combined_report_file) > 50:
         # Check if the .coverage data file is larger than the base file,
         # because coverage combine will always at least make the "empty" data
         # file even when there isn't any data to be combined.

scripts/Jenkinsfiles/bokchoy

@@ -14,14 +14,17 @@ def runBokchoyTests() {
             noTags: true, shallow: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker',
             refspec: git_refspec, url: "git@github.com:edx/${REPO_NAME}.git"]]]
         sh 'bash scripts/all-tests.sh'
+        stash includes: 'reports/bok_choy/*coverage*', name: "${TEST_SUITE}-reports-${SHARD}", allowEmpty: true
     }
 }
 
 def bokchoyTestCleanup() {
-    archiveArtifacts allowEmptyArchive: true, artifacts: 'test_root/log/**/*.log,test_root/log/**/*.png'
+    archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/bok_choy/*.coverage*,test_root/log/**/*.log,test_root/log/**/*.png'
     junit '**/reports/bok_choy/**/xunit.xml'
 }
 
+def shardCount = 25
+
 pipeline {
     agent { label "jenkins-worker" }
     options {
@@ -56,7 +59,7 @@ pipeline {
             steps {
                 script {
                     def parallel_stages = [:]
-                    for (int i = 1; i <= 25; i++) {
+                    for (int i = 1; i <= shardCount; i++) {
                         int index = i
                         parallel_stages["${index}"] = {
                             node('jenkins-worker') {
@@ -76,6 +79,32 @@ pipeline {
                 }
             }
         }
+        stage("Combine Coverage") {
+            when {
+                // Only combine coverage during context-collection
+                environment name: 'COLLECT_WHO_TESTS_WHAT', value: 'true'
+            }
+            steps {
+                script {
+                    sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) {
+                        checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: git_branch]],
+                            doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', honorRefspec: true,
+                            noTags: true, shallow: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker',
+                            refspec: git_refspec, url: "git@github.com:edx/${REPO_NAME}.git"]]]
+
+                        for (int i = 1; i <= shardCount; i++) {
+                            unstash "bok-choy-reports-${i}"
+                        }
+                        sh """
+                        export TEST_SUITE=bok-choy
+                        source scripts/jenkins-common.sh
+                        paver coverage --rcfile=common/test/acceptance/.coveragerc
+                        paver upload_coverage_to_s3
+                        """
+                    }
+                }
+            }
+        }
     }
     post {
         always {

scripts/Jenkinsfiles/python

@@ -175,6 +175,7 @@ pipeline {
             post {
                 always {
                     script {
+                        archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/${TEST_SUITE}.coverage'
                         publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true,
                             reportDir: 'reports/cover', reportFiles: 'index.html',
                             reportName: 'Coverage.py Report', reportTitles: ''])

scripts/generic-ci-tests.sh

@@ -175,10 +175,10 @@ case "$TEST_SUITE" in
     "bok-choy")
 
         PAVER_ARGS="-n $NUMBER_OF_BOKCHOY_THREADS"
-        if [[ -n "$WHO_TESTS_WHAT" ]]; then
+        if [[ -n "$FILTER_WHO_TESTS_WHAT" ]]; then
             PAVER_ARGS="$PAVER_ARGS --with-wtw=origin/master"
         fi
-        if [[ -n "$PYTEST_CONTEXTS" ]]; then
+        if [[ -n "$COLLECT_WHO_TESTS_WHAT" ]]; then
             PAVER_ARGS="$PAVER_ARGS --pytest-contexts --coveragerc=common/test/acceptance/.coveragerc"
         fi
         export BOKCHOY_HEADLESS=true

scripts/jenkins-report.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 set -e
+set -x
 
 # This script generates coverage and diff cover reports, and optionally
 # reports this data to codecov.io. The following environment variables must be
@@ -14,6 +15,10 @@ source scripts/jenkins-common.sh
 # Get the diff coverage and html reports for unit tests
 paver coverage -b $TARGET_BRANCH
 
+if [[ -n "$COLLECT_WHO_TESTS_WHAT" ]]; then
+    paver upload_coverage_to_s3
+fi
+
 # Test for the CodeCov API token
 if [ -z $CODE_COV_TOKEN ]; then
     echo "codecov.io API token not set."
@@ -27,7 +32,3 @@ fi
 # JUnit test reporter will fail the build
 # if it thinks test results are old
 touch `find . -name *.xml` || true
-
-if [[ -n "$PYTEST_CONTEXTS" ]]; then
-    paver upload_coverage_to_s3
-fi

scripts/unit-tests.sh

@@ -50,10 +50,10 @@ else
     PARALLEL="--processes=-1"
 fi
 
-if [[ -n "$WHO_TESTS_WHAT" ]]; then
-    PAVER_ARGS="$PAVER_ARGS --with-wtw"
+if [[ -n "$FILTER_WHO_TESTS_WHAT" ]]; then
+    PAVER_ARGS="$PAVER_ARGS --with-wtw=origin/master"
 fi
-if [[ -n "$PYTEST_CONTEXTS" ]]; then
+if [[ -n "$COLLECT_WHO_TESTS_WHAT" ]]; then
     PAVER_ARGS="$PAVER_ARGS --pytest-contexts"
 fi