From fdd212248a4658351766b8fc896d4d18ed6fe8be Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Wed, 24 Jul 2019 13:13:30 -0400 Subject: [PATCH 1/5] Check the test-suite specific coverage file, and upload it earlier in the process --- pavelib/tests.py | 6 ++++-- scripts/jenkins-report.sh | 9 +++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pavelib/tests.py b/pavelib/tests.py index 8598d2f2ba..f8bb8b7d81 100644 --- a/pavelib/tests.py +++ b/pavelib/tests.py @@ -331,12 +331,14 @@ def coverage(): report_dir = Env.REPORT_DIR rcfile = Env.PYTHON_COVERAGERC - if not (report_dir / '.coverage').isfile(): + combined_report_file = report_dir / '{}.coverage'.format(os.environ.get('TEST_SUITE', '')) + + if not combined_report_file.isfile(): # This may be that the coverage files were generated using -p, # try to combine them to the one file that we need. sh(u"coverage combine --rcfile={}".format(rcfile)) - if not os.path.getsize(report_dir / '.coverage') > 50: + if not os.path.getsize(combined_report_file) > 50: # Check if the .coverage data file is larger than the base file, # because coverage combine will always at least make the "empty" data # file even when there isn't any data to be combined. diff --git a/scripts/jenkins-report.sh b/scripts/jenkins-report.sh index 801920e9c1..904834e6b0 100755 --- a/scripts/jenkins-report.sh +++ b/scripts/jenkins-report.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -e +set -x # This script generates coverage and diff cover reports, and optionally # reports this data to codecov.io. The following environment variables must be @@ -14,6 +15,10 @@ source scripts/jenkins-common.sh # Get the diff coverage and html reports for unit tests paver coverage -b $TARGET_BRANCH +if [[ -n "$PYTEST_CONTEXTS" ]]; then + paver upload_coverage_to_s3 +fi + # Test for the CodeCov API token if [ -z $CODE_COV_TOKEN ]; then echo "codecov.io API token not set." @@ -27,7 +32,3 @@ fi # JUnit test reporter will fail the build # if it thinks test results are old touch `find . -name *.xml` || true - -if [[ -n "$PYTEST_CONTEXTS" ]]; then - paver upload_coverage_to_s3 -fi From a0e4adbaa53fd41313ade0c55cf21d257d3fd7c7 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Wed, 24 Jul 2019 13:37:30 -0400 Subject: [PATCH 2/5] Archive the combined python coverage file --- scripts/Jenkinsfiles/python | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/Jenkinsfiles/python b/scripts/Jenkinsfiles/python index 6950148b05..2fd561b679 100644 --- a/scripts/Jenkinsfiles/python +++ b/scripts/Jenkinsfiles/python @@ -175,6 +175,7 @@ pipeline { post { always { script { + archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/${TEST_SUITE}.coverage' publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true, reportDir: 'reports/cover', reportFiles: 'index.html', reportName: 'Coverage.py Report', reportTitles: '']) From 82c49f0db923168970295f8306dfceb86c162f5c Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Wed, 24 Jul 2019 13:47:46 -0400 Subject: [PATCH 3/5] Separate bokchoy coverage files by shard --- common/test/acceptance/.coveragerc | 2 +- pavelib/tests.py | 5 +++-- scripts/Jenkinsfiles/bokchoy | 33 ++++++++++++++++++++++++++++-- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/common/test/acceptance/.coveragerc b/common/test/acceptance/.coveragerc index 9d19a5b09a..116edaf636 100644 --- a/common/test/acceptance/.coveragerc +++ b/common/test/acceptance/.coveragerc @@ -1,5 +1,5 @@ [run] -data_file = reports/bok_choy/.coverage +data_file = reports/bok_choy/${TEST_SUITE}.coverage${SHARD} source = lms cms diff --git a/pavelib/tests.py b/pavelib/tests.py index f8bb8b7d81..3dfc43cfb9 100644 --- a/pavelib/tests.py +++ b/pavelib/tests.py @@ -322,14 +322,15 @@ def test(options, passthrough_options): @needs('pavelib.prereqs.install_coverage_prereqs') @cmdopts([ ("compare-branch=", "b", "Branch to compare against, defaults to origin/master"), + ("rcfile=", "c", "Coveragerc file to use, defaults to .coveragerc"), ]) @timed -def coverage(): +def coverage(options): """ Build the html, xml, and diff coverage reports """ report_dir = Env.REPORT_DIR - rcfile = Env.PYTHON_COVERAGERC + rcfile = getattr(options.coverage, 'rcfile', Env.PYTHON_COVERAGERC) combined_report_file = report_dir / '{}.coverage'.format(os.environ.get('TEST_SUITE', '')) diff --git a/scripts/Jenkinsfiles/bokchoy b/scripts/Jenkinsfiles/bokchoy index 13c77830be..667b04e59a 100644 --- a/scripts/Jenkinsfiles/bokchoy +++ b/scripts/Jenkinsfiles/bokchoy @@ -14,14 +14,17 @@ def runBokchoyTests() { noTags: true, shallow: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker', refspec: git_refspec, url: "git@github.com:edx/${REPO_NAME}.git"]]] sh 'bash scripts/all-tests.sh' + stash includes: 'reports/bok_choy/*coverage*', name: "${TEST_SUITE}-reports-${SHARD}", allowEmpty: true } } def bokchoyTestCleanup() { - archiveArtifacts allowEmptyArchive: true, artifacts: 'test_root/log/**/*.log,test_root/log/**/*.png' + archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/bok_choy/*.coverage*,test_root/log/**/*.log,test_root/log/**/*.png' junit '**/reports/bok_choy/**/xunit.xml' } +def shardCount = 25 + pipeline { agent { label "jenkins-worker" } options { @@ -56,7 +59,7 @@ pipeline { steps { script { def parallel_stages = [:] - for (int i = 1; i <= 25; i++) { + for (int i = 1; i <= shardCount; i++) { int index = i parallel_stages["${index}"] = { node('jenkins-worker') { @@ -76,6 +79,32 @@ pipeline { } } } + stage("Combine Coverage") { + when { + // Only combine coverage during context-collection + environment name: 'PYTEST_CONTEXTS', value: 'true' + } + steps { + script { + sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) { + checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: git_branch]], + doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', honorRefspec: true, + noTags: true, shallow: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker', + refspec: git_refspec, url: "git@github.com:edx/${REPO_NAME}.git"]]] + + for (int i = 1; i <= shardCount; i++) { + unstash "bok-choy-reports-${i}" + } + sh """ + export TEST_SUITE=bok-choy + source scripts/jenkins-common.sh + paver coverage --rcfile=common/test/acceptance/.coveragerc + paver upload_coverage_to_s3 + """ + } + } + } + } } post { always { From f638723b24634917edcb9c62cc7e0778fff7af5d Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Wed, 24 Jul 2019 13:59:19 -0400 Subject: [PATCH 4/5] Specify unittest wtw branch --- scripts/unit-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/unit-tests.sh b/scripts/unit-tests.sh index add85ebb8d..02be8fdb06 100755 --- a/scripts/unit-tests.sh +++ b/scripts/unit-tests.sh @@ -51,7 +51,7 @@ else fi if [[ -n "$WHO_TESTS_WHAT" ]]; then - PAVER_ARGS="$PAVER_ARGS --with-wtw" + PAVER_ARGS="$PAVER_ARGS --with-wtw=origin/master" fi if [[ -n "$PYTEST_CONTEXTS" ]]; then PAVER_ARGS="$PAVER_ARGS --pytest-contexts" From 68b83f763f7cbb5b5baac2e5f717ac16d330d812 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Fri, 26 Jul 2019 14:42:45 -0400 Subject: [PATCH 5/5] Rename who-tests-what environment variables --- scripts/Jenkinsfiles/bokchoy | 2 +- scripts/generic-ci-tests.sh | 4 ++-- scripts/jenkins-report.sh | 2 +- scripts/unit-tests.sh | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/Jenkinsfiles/bokchoy b/scripts/Jenkinsfiles/bokchoy index 667b04e59a..7bb64fc070 100644 --- a/scripts/Jenkinsfiles/bokchoy +++ b/scripts/Jenkinsfiles/bokchoy @@ -82,7 +82,7 @@ pipeline { stage("Combine Coverage") { when { // Only combine coverage during context-collection - environment name: 'PYTEST_CONTEXTS', value: 'true' + environment name: 'COLLECT_WHO_TESTS_WHAT', value: 'true' } steps { script { diff --git a/scripts/generic-ci-tests.sh b/scripts/generic-ci-tests.sh index 2acf3a5eaf..9f3fa82a0d 100755 --- a/scripts/generic-ci-tests.sh +++ b/scripts/generic-ci-tests.sh @@ -175,10 +175,10 @@ case "$TEST_SUITE" in "bok-choy") PAVER_ARGS="-n $NUMBER_OF_BOKCHOY_THREADS" - if [[ -n "$WHO_TESTS_WHAT" ]]; then + if [[ -n "$FILTER_WHO_TESTS_WHAT" ]]; then PAVER_ARGS="$PAVER_ARGS --with-wtw=origin/master" fi - if [[ -n "$PYTEST_CONTEXTS" ]]; then + if [[ -n "$COLLECT_WHO_TESTS_WHAT" ]]; then PAVER_ARGS="$PAVER_ARGS --pytest-contexts --coveragerc=common/test/acceptance/.coveragerc" fi export BOKCHOY_HEADLESS=true diff --git a/scripts/jenkins-report.sh b/scripts/jenkins-report.sh index 904834e6b0..b085826ee7 100755 --- a/scripts/jenkins-report.sh +++ b/scripts/jenkins-report.sh @@ -15,7 +15,7 @@ source scripts/jenkins-common.sh # Get the diff coverage and html reports for unit tests paver coverage -b $TARGET_BRANCH -if [[ -n "$PYTEST_CONTEXTS" ]]; then +if [[ -n "$COLLECT_WHO_TESTS_WHAT" ]]; then paver upload_coverage_to_s3 fi diff --git a/scripts/unit-tests.sh b/scripts/unit-tests.sh index 02be8fdb06..c060d39a0a 100755 --- a/scripts/unit-tests.sh +++ b/scripts/unit-tests.sh @@ -50,10 +50,10 @@ else PARALLEL="--processes=-1" fi -if [[ -n "$WHO_TESTS_WHAT" ]]; then +if [[ -n "$FILTER_WHO_TESTS_WHAT" ]]; then PAVER_ARGS="$PAVER_ARGS --with-wtw=origin/master" fi -if [[ -n "$PYTEST_CONTEXTS" ]]; then +if [[ -n "$COLLECT_WHO_TESTS_WHAT" ]]; then PAVER_ARGS="$PAVER_ARGS --pytest-contexts" fi