Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #7709 from edx/clytwynec/unique-dirs-for-test-results

Browse Source 
Clytwynec/unique dirs for test results
This commit is contained in:
Christine Lytwynec
2015-05-04 08:55:38 -04:00
parent 7a1060cf56 67ee646989
commit 4f5aeb3f49
6 changed files with 49 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
pavelib/paver_tests/test_paver_bok_choy_cmds.py
View File

@@ -14,19 +14,21 @@ class TestPaverBokChoyCmd(unittest.TestCase):
def _expected_command(self, expected_text_append, expected_default_store=None):
if expected_text_append:
expected_text_append = "/" + expected_text_append
shard = os.environ.get('SHARD')
expected_statement = (
"DEFAULT_STORE={default_store} "
"SCREENSHOT_DIR='{repo_dir}/test_root/log' "
"BOK_CHOY_HAR_DIR='{repo_dir}/test_root/log/hars' "
"SELENIUM_DRIVER_LOG_DIR='{repo_dir}/test_root/log' "
"SCREENSHOT_DIR='{repo_dir}/test_root/log{shard_str}' "
"BOK_CHOY_HAR_DIR='{repo_dir}/test_root/log{shard_str}/hars' "
"SELENIUM_DRIVER_LOG_DIR='{repo_dir}/test_root/log{shard_str}' "
"nosetests {repo_dir}/common/test/acceptance/tests{exp_text} "
"--with-xunit "
"--xunit-file={repo_dir}/reports/bok_choy/xunit.xml "
"--verbosity=2"
"--xunit-file={repo_dir}/reports/bok_choy{shard_str}/xunit.xml "
"--verbosity=2 "
).format(
default_store=expected_default_store,
repo_dir=REPO_DIR,
exp_text=expected_text_append,
shard_str='/shard_' + shard if shard else '',
)
return expected_statement.strip()

12
pavelib/tests.py
View File

@@ -212,6 +212,18 @@ def coverage(options):
call_task('diff_coverage', options=dict(options))
@task
@needs('pavelib.prereqs.install_prereqs')
def combine_coverage():
"""
Combine coverage reports.
"""
for directory in Env.LIB_TEST_DIRS + ['cms', 'lms']:
report_dir = Env.REPORT_DIR / directory
if report_dir.isdir():
sh("cd {} && coverage combine".format(report_dir))
@task
@needs('pavelib.prereqs.install_prereqs')
@cmdopts([

9
pavelib/utils/envs.py
View File

@@ -28,6 +28,15 @@ class Env(object):
BOK_CHOY_REPORT_DIR = REPORT_DIR / "bok_choy"
BOK_CHOY_COVERAGERC = BOK_CHOY_DIR / ".coveragerc"
# If set, put reports for run in "unique" directories.
# The main purpose of this is to ensure that the reports can be 'slurped'
# in the main jenkins flow job without overwriting the reports from other
# build steps. For local development/testing, this shouldn't be needed.
if os.environ.get("SHARD", None):
shard_str = "shard_{}".format(os.environ.get("SHARD"))
BOK_CHOY_REPORT_DIR = BOK_CHOY_REPORT_DIR / shard_str
BOK_CHOY_LOG_DIR = BOK_CHOY_LOG_DIR / shard_str
# For the time being, stubs are used by both the bok-choy and lettuce acceptance tests
# For this reason, the stubs package is currently located in the Django app called "terrain"
# where other lettuce configuration is stored.

13
pavelib/utils/test/suites/nose_suite.py
View File

@@ -21,6 +21,15 @@ class NoseTestSuite(TestSuite):
self.fail_fast = kwargs.get('fail_fast', False)
self.run_under_coverage = kwargs.get('with_coverage', True)
self.report_dir = Env.REPORT_DIR / self.root
# If set, put reports for run in "unique" directories.
# The main purpose of this is to ensure that the reports can be 'slurped'
# in the main jenkins flow job without overwriting the reports from other
# build steps. For local development/testing, this shouldn't be needed.
if os.environ.get("SHARD", None):
shard_str = "shard_{}".format(os.environ.get("SHARD"))
self.report_dir = self.report_dir / shard_str
self.test_id_dir = Env.TEST_DIR / self.root
self.test_ids = self.test_id_dir / 'noseids'
self.extra_args = kwargs.get('extra_args', '')
@@ -112,12 +121,14 @@ class SystemTestSuite(NoseTestSuite):
def cmd(self):
cmd = (
'./manage.py {system} test --verbosity={verbosity} '
'{test_id} {test_opts} --traceback --settings=test {extra}'.format(
'{test_id} {test_opts} --traceback --settings=test {extra} '
'--with-xunit --xunit-file={xunit_report}'.format(
system=self.root,
verbosity=self.verbosity,
test_id=self.test_id,
test_opts=self.test_options_flags,
extra=self.extra_args,
xunit_report=self.report_dir / "nosetests.xml",
)
)

21
scripts/all-tests.sh
View File

@@ -98,14 +98,12 @@ END
"unit")
case "$SHARD" in
"lms")
paver test_system -s lms --extra_args="--with-flaky" || { EXIT=1; }
paver coverage
SHARD=1 paver test_system -s lms --extra_args="--with-flaky" --cov_args="-p" || { EXIT=1; }
;;
"cms-js-commonlib")
paver test_system -s cms --extra_args="--with-flaky" || { EXIT=1; }
paver test_js --coverage --skip_clean || { EXIT=1; }
paver test_lib --skip_clean --extra_args="--with-flaky" || { EXIT=1; }
paver coverage
SHARD=1 paver test_system -s cms --extra_args="--with-flaky" --cov_args="-p" || { EXIT=1; }
SHARD=1 paver test_js --coverage --skip_clean || { EXIT=1; }
SHARD=1 paver test_lib --skip_clean --extra_args="--with-flaky" --cov_args="-p" || { EXIT=1; }
;;
*)
paver test --extra_args="--with-flaky"
@@ -210,15 +208,4 @@ END
END
;;
esac
# Move the reports to a directory that is unique to the shard
# so that when they are 'slurped' to the main flow job, they
# do not conflict with and overwrite reports from other shards.
mv reports/ reports_tmp/
mkdir -p reports/${TEST_SUITE}/${SHARD}
mv reports_tmp/* reports/${TEST_SUITE}/${SHARD}
rm -r reports_tmp/
exit $EXIT
;;
esac

7
scripts/jenkins-report.sh
View File

@@ -1,8 +1,11 @@
#!/usr/bin/env bash
source scripts/jenkins-common.sh
# Run coverage again to get the diff coverage report
paver diff_coverage
# Combine the data files that were generated using -p
paver combine_coverage
# Get the diff coverage and html reports for unit tests
paver coverage
# JUnit test reporter will fail the build
# if it thinks test results are old