From 12ca7b8ab706156eb9195b948f7b0f5264b6104c Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 16 Jul 2019 07:05:53 -0400 Subject: [PATCH 1/2] Name coverage data files so they won't overwrite each other. --- .coveragerc | 2 +- pavelib/utils/test/suites/suite.py | 3 +++ scripts/unit-tests.sh | 18 +++++++++--------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.coveragerc b/.coveragerc index 44b8c467a2..596d280140 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,6 @@ # .coveragerc for edx-platform [run] -data_file = reports/.coverage +data_file = reports/${TEST_SUITE}.coverage source = cms common/djangoapps diff --git a/pavelib/utils/test/suites/suite.py b/pavelib/utils/test/suites/suite.py index f6c9bea561..f23e371057 100644 --- a/pavelib/utils/test/suites/suite.py +++ b/pavelib/utils/test/suites/suite.py @@ -4,6 +4,7 @@ A class used for defining and running test suites """ from __future__ import absolute_import, print_function +import os import subprocess import sys @@ -93,6 +94,8 @@ class TestSuite(object): sys.stdout.write(msg) sys.stdout.flush() + if 'TEST_SUITE' not in os.environ: + os.environ['TEST_SUITE'] = self.root.replace("/", "_") kwargs = {'shell': True, 'cwd': None} process = None diff --git a/scripts/unit-tests.sh b/scripts/unit-tests.sh index 93aa2b0bda..c70ef2280e 100755 --- a/scripts/unit-tests.sh +++ b/scripts/unit-tests.sh @@ -56,15 +56,15 @@ case "${TEST_SUITE}" in case "$SHARD" in "all") paver test_system -s lms --disable_capture ${PAVER_ARGS} ${PARALLEL} 2> lms-tests.log - mv reports/.coverage reports/.coverage.lms + mv reports/${TEST_SUITE}.coverage reports/.coverage.lms ;; [1-9]) paver test_system -s lms --disable_capture --eval-attr="shard==$SHARD" ${PAVER_ARGS} ${PARALLEL} 2> lms-tests.${SHARD}.log - mv reports/.coverage reports/.coverage.lms.${SHARD} + mv reports/${TEST_SUITE}.coverage reports/.coverage.lms.${SHARD} ;; 10|"noshard") paver test_system -s lms --disable_capture --eval-attr="shard>=$SHARD or not shard" ${PAVER_ARGS} ${PARALLEL} 2> lms-tests.10.log - mv reports/.coverage reports/.coverage.lms.10 + mv reports/${TEST_SUITE}.coverage reports/.coverage.lms.10 ;; *) # If no shard is specified, rather than running all tests, create an empty xunit file. This is a @@ -81,15 +81,15 @@ case "${TEST_SUITE}" in case "$SHARD" in "all") paver test_system -s cms --disable_capture ${PAVER_ARGS} ${PARALLEL} 2> cms-tests.log - mv reports/.coverage reports/.coverage.cms + mv reports/${TEST_SUITE}.coverage reports/.coverage.cms ;; 1) paver test_system -s cms --disable_capture --eval-attr="shard==$SHARD" ${PAVER_ARGS} 2> cms-tests.${SHARD}.log - mv reports/.coverage reports/.coverage.cms.${SHARD} + mv reports/${TEST_SUITE}.coverage reports/.coverage.cms.${SHARD} ;; 2|"noshard") paver test_system -s cms --disable_capture --eval-attr="shard>=$SHARD or not shard" ${PAVER_ARGS} 2> cms-tests.2.log - mv reports/.coverage reports/.coverage.cms.2 + mv reports/${TEST_SUITE}.coverage reports/.coverage.cms.2 ;; *) # If no shard is specified, rather than running all tests, create an empty xunit file. This is a @@ -106,15 +106,15 @@ case "${TEST_SUITE}" in case "$SHARD" in "all") paver test_lib --disable_capture ${PAVER_ARGS} ${PARALLEL} 2> common-tests.log - mv reports/.coverage reports/.coverage.commonlib + mv reports/${TEST_SUITE}.coverage reports/.coverage.commonlib ;; [1-2]) paver test_lib -l common/lib/xmodule --disable_capture --eval-attr="shard==$SHARD" ${PAVER_ARGS} 2> common-tests.${SHARD}.log - mv reports/.coverage reports/.coverage.commonlib.${SHARD} + mv reports/${TEST_SUITE}.coverage reports/.coverage.commonlib.${SHARD} ;; 3|"noshard") paver test_lib --disable_capture --eval-attr="shard>=$SHARD or not shard" ${PAVER_ARGS} 2> common-tests.3.log - mv reports/.coverage reports/.coverage.commonlib.3 + mv reports/${TEST_SUITE}.coverage reports/.coverage.commonlib.3 ;; *) # If no shard is specified, rather than running all tests, create an empty xunit file. This is a From 2234aeb1abcf7d12aaddec440dff20fede36bc98 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 22 Jul 2019 16:24:55 -0400 Subject: [PATCH 2/2] Coverage et al that support who-tests-what --- requirements/constraints.txt | 3 +++ requirements/edx/base.txt | 6 +++--- requirements/edx/coverage.in | 2 +- requirements/edx/coverage.txt | 2 +- requirements/edx/development.txt | 14 +++++++++----- requirements/edx/testing.in | 6 +++++- requirements/edx/testing.txt | 14 +++++++++----- 7 files changed, 31 insertions(+), 16 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index b884cce929..59b25ee513 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -72,3 +72,6 @@ ruamel.ordereddict; python_version == "2.7" # 1.16.1 requires djangorestframework>=3.8 drf-yasg==1.16 + +# 2.0.0 is a dummy package, because faulthandler has been incorporated into pytest 5.0 +pytest-faulthandler<2.0.0 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 968608891b..db24e05dc6 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -30,7 +30,7 @@ asn1crypto==0.24.0 attrs==17.4.0 babel==1.3 backports.functools-lru-cache==1.5 # via soupsieve -beautifulsoup4==4.7.1 # via pynliner +beautifulsoup4==4.8.0 # via pynliner billiard==3.3.0.23 # via celery bleach==2.1.4 boto3==1.4.8 @@ -105,7 +105,7 @@ edx-django-oauth2-provider==1.3.5 edx-django-release-util==0.3.1 edx-django-sites-extensions==2.3.1 edx-django-utils==2.0.0 -edx-drf-extensions==2.3.5 +edx-drf-extensions==2.3.6 edx-enterprise==1.7.3 edx-i18n-tools==0.4.8 edx-milestones==0.2.3 @@ -234,7 +234,7 @@ soupsieve==1.9.2 # via beautifulsoup4 sqlparse==0.3.0 staff-graded-xblock==0.3 stevedore==1.30.1 -super-csv==0.7 +super-csv==0.7.1 sympy==1.4 testfixtures==6.10.0 # via edx-enterprise tincan==0.0.5 # via edx-enterprise diff --git a/requirements/edx/coverage.in b/requirements/edx/coverage.in index a0c036431a..5faf6d8fe0 100644 --- a/requirements/edx/coverage.in +++ b/requirements/edx/coverage.in @@ -12,6 +12,6 @@ -c ../constraints.txt -coverage==4.4 # Code coverage testing for Python +coverage==5.0a6 # Code coverage testing for Python diff-cover==0.9.8 # Automatically find diff lines that need test coverage six==1.11.0 # Pinned because diff-cover needs it, but later transifex-client says ==1.11.0 diff --git a/requirements/edx/coverage.txt b/requirements/edx/coverage.txt index 09cbd1446f..47f4fd9335 100644 --- a/requirements/edx/coverage.txt +++ b/requirements/edx/coverage.txt @@ -4,7 +4,7 @@ # # make upgrade # -coverage==4.4 +coverage==5.0a6 diff-cover==0.9.8 inflect==2.1.0 # via jinja2-pluralize jinja2-pluralize==0.3.0 # via diff-cover diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 09941c56d4..f56a572dc7 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -36,7 +36,7 @@ atomicwrites==1.3.0 attrs==17.4.0 babel==1.3 backports.functools-lru-cache==1.5 -beautifulsoup4==4.7.1 +beautifulsoup4==4.8.0 before-after==1.0.1 billiard==3.3.0.23 bleach==2.1.4 @@ -59,7 +59,8 @@ configparser==3.7.4 contextlib2==0.5.5 coreapi==2.3.3 coreschema==0.0.4 -coverage==4.4 +coverage==5.0a6 +git+https://github.com/nedbat/coverage_pytest_plugin.git@29de030251471e200ff255eb9e549218cd60e872#egg=coverage_pytest_plugin==0.0 git+https://github.com/edx/crowdsourcehinter.git@518605f0a95190949fe77bd39158450639e2e1dc#egg=crowdsourcehinter-xblock==0.1 cryptography==2.7 cssselect==1.0.3 @@ -124,7 +125,7 @@ edx-django-oauth2-provider==1.3.5 edx-django-release-util==0.3.1 edx-django-sites-extensions==2.3.1 edx-django-utils==2.0.0 -edx-drf-extensions==2.3.5 +edx-drf-extensions==2.3.6 edx-enterprise==1.7.3 edx-i18n-tools==0.4.8 edx-lint==1.3.0 @@ -150,6 +151,7 @@ event-tracking==0.2.9 execnet==1.6.0 factory_boy==2.8.1 faker==2.0.0 +faulthandler==3.1 feedparser==5.1.3 filelock==3.0.12 firebase-token-generator==1.3.2 @@ -256,8 +258,9 @@ pyquery==1.4.0 pysqlite==2.8.3 ; python_version == "2.7" pysrt==1.1.1 pytest-attrib==0.1.3 -pytest-cov==2.7.1 +git+https://github.com/nedbat/pytest-cov.git@nedbat/cov5-combine#egg=pytest-cov==0.0 pytest-django==3.5.1 +pytest-faulthandler==1.6.0 pytest-forked==1.0.2 pytest-randomly==1.2.3 pytest-xdist==1.29.0 @@ -307,7 +310,7 @@ sphinxcontrib-websupport==1.1.2 # via sphinx sqlparse==0.3.0 staff-graded-xblock==0.3 stevedore==1.30.1 -super-csv==0.7 +super-csv==0.7.1 sympy==1.4 testfixtures==6.10.0 text-unidecode==1.2 @@ -319,6 +322,7 @@ transifex-client==0.13.6 typing==3.7.4 unicodecsv==0.14.1 unidecode==1.1.1 +unidiff==0.5.5 uritemplate==3.0.0 urllib3==1.23 user-util==0.1.5 diff --git a/requirements/edx/testing.in b/requirements/edx/testing.in index e72e3d6e0c..742525f6ad 100644 --- a/requirements/edx/testing.in +++ b/requirements/edx/testing.in @@ -37,7 +37,9 @@ pyquery # jQuery-like API for retrieving fragments of HTML and pysqlite # DB-API 2.0 interface for SQLite 3.x (used as the relational database for most tests) pytest # Testing framework pytest-attrib # Select tests based on attributes -pytest-cov # pytest plugin for measuring code coverage. +git+https://github.com/nedbat/pytest-cov.git@nedbat/cov5-combine#egg=pytest-cov==0.0 +git+https://github.com/nedbat/coverage_pytest_plugin.git@29de030251471e200ff255eb9e549218cd60e872#egg=coverage_pytest_plugin==0.0 +pytest-faulthandler # Report on serious crashes pytest-django # Django support for pytest pytest-randomly # pytest plugin to randomly order tests pytest-xdist # Parallel execution of tests on multiple CPU cores or hosts @@ -48,3 +50,5 @@ testfixtures # Provides a LogCapture utility used by several tests tox # virtualenv management for tests tox-battery # Makes tox aware of requirements file changes transifex-client # Command-line interface for the Transifex localization service +unidiff # Required by coverage_pytest_plugin + diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index a3c7bc101f..c6f2fdab4b 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -34,7 +34,7 @@ atomicwrites==1.3.0 # via pytest attrs==17.4.0 babel==1.3 backports.functools-lru-cache==1.5 -beautifulsoup4==4.7.1 +beautifulsoup4==4.8.0 before-after==1.0.1 billiard==3.3.0.23 bleach==2.1.4 @@ -57,7 +57,8 @@ configparser==3.7.4 # via entrypoints, flake8, importlib-metadata, pylint contextlib2==0.5.5 # via importlib-metadata coreapi==2.3.3 coreschema==0.0.4 -coverage==4.4 +coverage==5.0a6 +git+https://github.com/nedbat/coverage_pytest_plugin.git@29de030251471e200ff255eb9e549218cd60e872#egg=coverage_pytest_plugin==0.0 git+https://github.com/edx/crowdsourcehinter.git@518605f0a95190949fe77bd39158450639e2e1dc#egg=crowdsourcehinter-xblock==0.1 cryptography==2.7 cssselect==1.0.3 @@ -120,7 +121,7 @@ edx-django-oauth2-provider==1.3.5 edx-django-release-util==0.3.1 edx-django-sites-extensions==2.3.1 edx-django-utils==2.0.0 -edx-drf-extensions==2.3.5 +edx-drf-extensions==2.3.6 edx-enterprise==1.7.3 edx-i18n-tools==0.4.8 edx-lint==1.3.0 @@ -145,6 +146,7 @@ event-tracking==0.2.9 execnet==1.6.0 # via pytest-xdist factory_boy==2.8.1 faker==2.0.0 # via factory-boy +faulthandler==3.1 # via pytest-faulthandler feedparser==5.1.3 filelock==3.0.12 # via tox firebase-token-generator==1.3.2 @@ -247,8 +249,9 @@ pyquery==1.4.0 pysqlite==2.8.3 ; python_version == "2.7" pysrt==1.1.1 pytest-attrib==0.1.3 -pytest-cov==2.7.1 +git+https://github.com/nedbat/pytest-cov.git@nedbat/cov5-combine#egg=pytest-cov==0.0 pytest-django==3.5.1 +pytest-faulthandler==1.6.0 pytest-forked==1.0.2 # via pytest-xdist pytest-randomly==1.2.3 pytest-xdist==1.29.0 @@ -294,7 +297,7 @@ soupsieve==1.9.2 sqlparse==0.3.0 staff-graded-xblock==0.3 stevedore==1.30.1 -super-csv==0.7 +super-csv==0.7.1 sympy==1.4 testfixtures==6.10.0 text-unidecode==1.2 # via faker @@ -306,6 +309,7 @@ transifex-client==0.13.6 typing==3.7.4 # via flake8 unicodecsv==0.14.1 unidecode==1.1.1 # via python-slugify +unidiff==0.5.5 uritemplate==3.0.0 urllib3==1.23 user-util==0.1.5