From 6c69b6d435e6a9ee8ea8d42d8f62d4b379d0697e Mon Sep 17 00:00:00 2001 From: Manjinder Singh <49171515+jinder1s@users.noreply.github.com> Date: Thu, 2 Jan 2020 10:01:52 -0500 Subject: [PATCH] Adding code to output pytest warnings. (#22570) * Added pytest-json-report plugin - modifying app-opts in setup.cfg - adding hook to all conftest.py files in repo - setting report to be saved to test_root/log/warnings.json - Writing custom logic to save json report to avoid overwrite if pytest called twice This was created to allow us to easily parse through test warnings in jenkins --- cms/conftest.py | 3 + common/lib/conftest.py | 2 + common/test/conftest.py | 1 + conftest.py | 3 + openedx/core/process_warnings.py | 246 ++++++++++++++++++++++++++++++ openedx/core/pytest_hooks.py | 79 ++++++++++ openedx/core/write_to_html.py | 98 ++++++++++++ pavelib/paver_tests/conftest.py | 2 + requirements/constraints.txt | 6 + requirements/edx-sandbox/py35.txt | 2 +- requirements/edx/base.in | 2 +- requirements/edx/base.txt | 10 +- requirements/edx/coverage.in | 1 + requirements/edx/coverage.txt | 4 + requirements/edx/development.txt | 13 +- requirements/edx/testing.in | 1 + requirements/edx/testing.txt | 13 +- scripts/Jenkinsfiles/python | 20 ++- setup.cfg | 2 +- 19 files changed, 488 insertions(+), 20 deletions(-) create mode 100644 openedx/core/process_warnings.py create mode 100644 openedx/core/pytest_hooks.py create mode 100644 openedx/core/write_to_html.py diff --git a/cms/conftest.py b/cms/conftest.py index 03a0490d7b..92a5c06163 100644 --- a/cms/conftest.py +++ b/cms/conftest.py @@ -12,6 +12,9 @@ import os import contracts import pytest +from openedx.core.pytest_hooks import pytest_json_modifyreport # pylint: disable=unused-import +from openedx.core.pytest_hooks import pytest_sessionfinish # pylint: disable=unused-import + # Patch the xml libs before anything else. from safe_lxml import defuse_xml_libs diff --git a/common/lib/conftest.py b/common/lib/conftest.py index 0c19cc206a..7bd2d497ca 100644 --- a/common/lib/conftest.py +++ b/common/lib/conftest.py @@ -7,6 +7,8 @@ import pytest from safe_lxml import defuse_xml_libs +from openedx.core.pytest_hooks import pytest_configure # pylint: disable=unused-import + defuse_xml_libs() diff --git a/common/test/conftest.py b/common/test/conftest.py index a23d052351..055da4ea8a 100644 --- a/common/test/conftest.py +++ b/common/test/conftest.py @@ -3,6 +3,7 @@ # Patch the xml libs before anything else. +from openedx.core.pytest_hooks import pytest_configure # pylint: disable=unused-import from safe_lxml import defuse_xml_libs defuse_xml_libs() diff --git a/conftest.py b/conftest.py index 8d5b764909..57ccae5296 100644 --- a/conftest.py +++ b/conftest.py @@ -10,6 +10,9 @@ import pytest # avoid duplicating the implementation from cms.conftest import _django_clear_site_cache, pytest_configure # pylint: disable=unused-import +from openedx.core.pytest_hooks import pytest_json_modifyreport # pylint: disable=unused-import +from openedx.core.pytest_hooks import pytest_sessionfinish # pylint: disable=unused-import + # When using self.assertEquals, diffs are truncated. We don't want that, always # show the whole diff. diff --git a/openedx/core/process_warnings.py b/openedx/core/process_warnings.py new file mode 100644 index 0000000000..6c749e76f1 --- /dev/null +++ b/openedx/core/process_warnings.py @@ -0,0 +1,246 @@ +""" +Script to process pytest warnings output by pytest-json-report plugin and output it as a html +""" +from __future__ import absolute_import +from __future__ import print_function +import json +import os +import io +import re +import argparse +from collections import Counter +import pandas as pd + +from write_to_html import ( + HtmlOutlineWriter, +) # noqa pylint: disable=import-error,useless-suppression + +columns = [ + "message", + "category", + "filename", + "lineno", + "high_location", + "label", + "num", + "deprecated", +] +columns_index_dict = {key: index for index, key in enumerate(columns)} + + +def seperate_warnings_by_location(warnings_data): + """ + Warnings originate from multiple locations, this function takes in list of warning objects + and separates them based on their filename location + """ + + # first create regex for each n file location + warnings_locations = { + ".*/python\d\.\d/site-packages/.*\.py": "python", # noqa pylint: disable=W1401 + ".*/edx-platform/lms/.*\.py": "lms", # noqa pylint: disable=W1401 + ".*/edx-platform/openedx/.*\.py": "openedx", # noqa pylint: disable=W1401 + ".*/edx-platform/cms/.*\.py": "cms", # noqa pylint: disable=W1401 + ".*/edx-platform/common/.*\.py": "common", # noqa pylint: disable=W1401 + } + + # separate into locations flow: + # - iterate through each wanring_object, see if its filename matches any regex in warning locations. + # - If so, change high_location index on warnings_object to location name + for warnings_object in warnings_data: + warning_origin_located = False + for key in warnings_locations: + if ( + re.search(key, warnings_object[columns_index_dict["filename"]]) + is not None + ): + warnings_object[ + columns_index_dict["high_location"] + ] = warnings_locations[key] + warning_origin_located = True + break + if not warning_origin_located: + warnings_object[columns_index_dict["high_location"]] = "other" + return warnings_data + + +def convert_warning_dict_to_list(warning_dict): + """ + converts our data dict into our defined list based on columns defined at top of this file + """ + output = [] + for column in columns: + if column in warning_dict: + output.append(warning_dict[column]) + else: + output.append(None) + output[columns_index_dict["num"]] = 1 + return output + + +def read_warning_data(dir_path): + """ + During test runs in jenkins, multiple warning json files are output. This function finds all files + and aggregates the warnings in to one large list + """ + # pdb.set_trace() + dir_path = os.path.expanduser(dir_path) + # find all files that exist in given directory + files_in_dir = [ + f for f in os.listdir(dir_path) if os.path.isfile(os.path.join(dir_path, f)) + ] + warnings_files = [] + + # TODO(jinder): currently this is hard-coded in, maybe create a constants file with info + # THINK(jinder): but creating file for one constant seems overkill + warnings_file_name_regex = ( + "pytest_warnings_?\d*\.json" # noqa pylint: disable=W1401 + ) + + # iterate through files_in_dir and see if they match our know file name pattern + for temp_file in files_in_dir: + if re.search(warnings_file_name_regex, temp_file) is not None: + warnings_files.append(temp_file) + + # go through each warning file and aggregate warnings into warnings_data + warnings_data = [] + for temp_file in warnings_files: + with io.open(os.path.expanduser(dir_path + "/" + temp_file), "r") as read_file: + json_input = json.load(read_file) + if "warnings" in json_input: + data = [ + convert_warning_dict_to_list(warning_dict) + for warning_dict in json_input["warnings"] + ] + warnings_data.extend(data) + else: + print(temp_file) + return warnings_data + + +def compress_similar_warnings(warnings_data): + """ + find all warnings that are exactly the same, count them, and return set with count added to each warning + """ + tupled_data = [tuple(data) for data in warnings_data] + test_counter = Counter(tupled_data) + output = [list(value) for value in test_counter.keys()] + for data_object in output: + data_object[columns_index_dict["num"]] = test_counter[tuple(data_object)] + return output + + +def process_warnings_json(dir_path): + """ + Master function to process through all warnings and output a dict + + dict structure: + { + location: [{warning text: {file_name: warning object}}] + } + + flow: + - Aggregate data from all warning files + - Separate warnings by deprecated vs non deprecated(has word deprecate in it) + - Further categorize warnings + - Return output + Possible Error/enhancement: there might be better ways to separate deprecates vs + non-deprecated warnings + """ + warnings_data = read_warning_data(dir_path) + for warnings_object in warnings_data: + warnings_object[columns_index_dict["deprecated"]] = bool( + "deprecated" in warnings_object[columns_index_dict["message"]] + ) + warnings_data = seperate_warnings_by_location(warnings_data) + compressed_warnings_data = compress_similar_warnings(warnings_data) + return compressed_warnings_data + + +def group_and_sort_by_sumof(dataframe, group, sort_by): + groups_by = dataframe.groupby(group) + temp_list_to_sort = [(key, value, value[sort_by].sum()) for key, value in groups_by] + # sort by count + return sorted(temp_list_to_sort, key=lambda x: -x[2]) + + +def write_html_report(warnings_dataframe, html_path): + """ + converts from panda dataframe to our html + """ + html_path = os.path.expanduser(html_path) + if "/" in html_path: + location_of_last_dir = html_path.rfind("/") + dir_path = html_path[:location_of_last_dir] + os.makedirs(dir_path, exist_ok=True) + with io.open(html_path, "w") as fout: + html_writer = HtmlOutlineWriter(fout) + category_sorted_by_count = group_and_sort_by_sumof( + warnings_dataframe, "category", "num" + ) + for category, group_in_category, category_count in category_sorted_by_count: + # xss-lint: disable=python-wrap-html + html = u'{category}, count: {count} '.format( + category=category, count=category_count + ) + html_writer.start_section(html, klass=u"category") + locations_sorted_by_count = group_and_sort_by_sumof( + group_in_category, "high_location", "num" + ) + + for ( + location, + group_in_location, + location_count, + ) in locations_sorted_by_count: + # xss-lint: disable=python-wrap-html + html = u'{location}, count: {count} '.format( + location=location, count=location_count + ) + html_writer.start_section(html, klass=u"location") + message_group_sorted_by_count = group_and_sort_by_sumof( + group_in_location, "message", "num" + ) + for ( + message, + message_group, + message_count, + ) in message_group_sorted_by_count: + # xss-lint: disable=python-wrap-html + html = u'{warning_text}, count: {count} '.format( + warning_text=message, count=message_count + ) + html_writer.start_section(html, klass=u"warning_text") + # warnings_object[location][warning_text] is a list + for _, warning in message_group.iterrows(): + # xss-lint: disable=python-wrap-html + html = u'{warning_file_path} '.format( + warning_file_path=warning["filename"] + ) + html_writer.start_section(html, klass=u"warning") + # xss-lint: disable=python-wrap-html + html = u'

lineno: {lineno}

'.format( + lineno=warning["lineno"] + ) + html_writer.write(html) + # xss-lint: disable=python-wrap-html + html = u'

num_occur: {num}

'.format( + num=warning["num"] + ) + html_writer.write(html) + + html_writer.end_section() + html_writer.end_section() + html_writer.end_section() + html_writer.end_section() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Process and categorize pytest warnings and output html report." + ) + parser.add_argument("--dir-path", default="test_root/log") + parser.add_argument("--html-path", default="test_html.html") + args = parser.parse_args() + data_output = process_warnings_json(args.dir_path) + data_dataframe = pd.DataFrame(data=data_output, columns=columns) + write_html_report(data_dataframe, args.html_path) diff --git a/openedx/core/pytest_hooks.py b/openedx/core/pytest_hooks.py new file mode 100644 index 0000000000..0ec2e3e996 --- /dev/null +++ b/openedx/core/pytest_hooks.py @@ -0,0 +1,79 @@ +""" +Module to put all pytest hooks that modify pytest behaviour +""" +import os +import io +import json + + +def pytest_json_modifyreport(json_report): + """ + - The function is called by pytest-json-report plugin to only output warnings in json format. + - Everything else is removed due to it already being saved by junitxml + - --json-omit flag in does not allow us to remove everything but the warnings + - (the environment metadata is one example of unremoveable data) + - The json warning outputs are meant to be read by jenkins + """ + warnings_flag = "warnings" + if warnings_flag in json_report: + warnings = json_report[warnings_flag] + json_report.clear() + json_report[warnings_flag] = warnings + else: + json_report = {} + return json_report + + +def create_file_name(dir_path, file_name_postfix, num=0): + """ + Used to create file name with this given + structure: TEST_SUITE + "_" + file_name_postfix + "_ " + num.json + The env variable TEST_SUITE is set in jenkinsfile + + This was necessary cause Pytest is run multiple times and we need to make sure old pytest + warning json files are not being overwritten. + """ + name = dir_path + "/" + if "TEST_SUITE" in os.environ: + name += os.environ["TEST_SUITE"] + "_" + name += file_name_postfix + if num != 0: + name += "_" + str(num) + return name + ".json" + + +def pytest_sessionfinish(session): + """ + Since multiple pytests are running, + this makes sure warnings from different run are not overwritten + """ + dir_path = "test_root/log" + file_name_postfix = "pytest_warnings" + num = 0 + # to make sure this doesn't loop forever, putting a maximum + while ( + os.path.isfile(create_file_name(dir_path, file_name_postfix, num)) and num < 100 + ): + num += 1 + + report = session.config._json_report.report # noqa pylint: disable=protected-access + + with io.open(create_file_name(dir_path, file_name_postfix, num), "w") as outfile: + json.dump(report, outfile) + + +class DeferPlugin(object): + """Simple plugin to defer pytest-xdist hook functions.""" + + def pytest_json_modifyreport(self, json_report): + """standard xdist hook function. + """ + return pytest_json_modifyreport(json_report) + + def pytest_sessionfinish(self, session): + return pytest_sessionfinish(session) + + +def pytest_configure(config): + if config.pluginmanager.hasplugin("json-report"): + config.pluginmanager.register(DeferPlugin()) diff --git a/openedx/core/write_to_html.py b/openedx/core/write_to_html.py new file mode 100644 index 0000000000..108ea179a9 --- /dev/null +++ b/openedx/core/write_to_html.py @@ -0,0 +1,98 @@ +""" +Class used to write pytest warning data into html format +""" +import textwrap +import six + + +class HtmlOutlineWriter(object): + """ + writer to handle html writing + """ + HEAD = textwrap.dedent( + u""" + + + + + + + + """ + ) + + SECTION_START = textwrap.dedent( + u"""\ +
+ + +
+ """ + ) + + SECTION_END = six.u("
") + + def __init__(self, fout): + self.fout = fout + self.section_id = 0 + self.fout.write(self.HEAD) + + def start_section(self, html, klass=None): + self.fout.write( + self.SECTION_START.format(id=self.section_id, html=html, klass=klass or "",) + ) + self.section_id += 1 + + def end_section(self): + self.fout.write(self.SECTION_END) + + def write(self, html): + self.fout.write(html) diff --git a/pavelib/paver_tests/conftest.py b/pavelib/paver_tests/conftest.py index 214a35e3fe..e6660c0cda 100644 --- a/pavelib/paver_tests/conftest.py +++ b/pavelib/paver_tests/conftest.py @@ -9,6 +9,8 @@ from shutil import rmtree import pytest from pavelib.utils.envs import Env +from openedx.core.pytest_hooks import pytest_json_modifyreport # pylint: disable=unused-import +from openedx.core.pytest_hooks import pytest_sessionfinish # pylint: disable=unused-import @pytest.fixture(autouse=True, scope='session') diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 60f9084621..1de3082f75 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -58,3 +58,9 @@ python3-saml==1.5.0 # transifex-client 0.13.5 and 0.13.6 pin six and urllib3 to old versions needlessly # https://github.com/transifex/transifex-client/issues/252 transifex-client==0.13.4 + +# moving constraint from base.in to constraints.txt +python-dateutil<2.6.0 + +#higher releases require python>3.5 +pandas<0.25.0 diff --git a/requirements/edx-sandbox/py35.txt b/requirements/edx-sandbox/py35.txt index 458f9b2e50..3db7b19f5d 100644 --- a/requirements/edx-sandbox/py35.txt +++ b/requirements/edx-sandbox/py35.txt @@ -22,7 +22,7 @@ nltk==3.4.5 numpy==1.16.5 pycparser==2.19 pyparsing==2.2.0 -python-dateutil==2.8.1 # via matplotlib +python-dateutil==2.5.3 # via matplotlib pytz==2019.3 # via matplotlib random2==1.0.1 scipy==1.2.1 diff --git a/requirements/edx/base.in b/requirements/edx/base.in index dedc3daee8..97d5192bfa 100644 --- a/requirements/edx/base.in +++ b/requirements/edx/base.in @@ -124,7 +124,7 @@ pyjwkest==1.3.2 PyJWT==1.5.2 pymongo # MongoDB driver pynliner # Inlines CSS styles into HTML for email notifications -python-dateutil==2.4 +python-dateutil python-Levenshtein python3-openid ; python_version>='3' python3-saml diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 16f4edbbd7..6ed1d427d2 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -21,7 +21,7 @@ -e common/lib/xmodule amqp==1.4.9 # via kombu analytics-python==1.2.9 -aniso8601==8.0.0 # via tincan +aniso8601==8.0.0 # via edx-tincan-py35 anyjson==0.3.3 # via kombu appdirs==1.4.3 # via fs argh==0.26.2 @@ -104,19 +104,20 @@ edx-django-release-util==0.3.2 edx-django-sites-extensions==2.4.2 edx-django-utils==2.0.2 edx-drf-extensions==2.4.5 -edx-enterprise==2.0.35 +edx-enterprise==2.0.36 edx-i18n-tools==0.5.0 edx-milestones==0.2.6 edx-oauth2-provider==1.3.1 edx-opaque-keys[django]==2.0.1 edx-organizations==2.2.0 edx-proctoring-proctortrack==1.0.5 -edx-proctoring==2.2.3 +edx-proctoring==2.2.4 edx-rbac==1.0.5 # via edx-enterprise edx-rest-api-client==1.9.2 edx-search==1.2.2 edx-sga==0.10.0 edx-submissions==3.0.3 +edx-tincan-py35==0.0.5 # via edx-enterprise edx-user-state-client==1.1.2 edx-when==0.5.2 edxval==1.1.33 @@ -191,7 +192,7 @@ pymongo==3.9.0 pynliner==0.8.0 pyparsing==2.2.0 # via pycontracts pysrt==1.1.1 -python-dateutil==2.4.0 +python-dateutil==2.5.3 python-levenshtein==0.12.0 python-memcached==1.59 python-slugify==4.0.0 # via code-annotations @@ -232,7 +233,6 @@ super-csv==0.9.6 sympy==1.5 testfixtures==6.10.3 # via edx-enterprise text-unidecode==1.3 # via python-slugify -tincan==0.0.5 # via edx-enterprise unicodecsv==0.14.1 uritemplate==3.0.1 # via coreapi, drf-yasg urllib3==1.25.7 diff --git a/requirements/edx/coverage.in b/requirements/edx/coverage.in index e52a152f87..dd0104c1dc 100644 --- a/requirements/edx/coverage.in +++ b/requirements/edx/coverage.in @@ -14,3 +14,4 @@ coverage # Code coverage testing for Python diff-cover # Automatically find diff lines that need test coverage +pandas # Used to process warnings generated by pytest diff --git a/requirements/edx/coverage.txt b/requirements/edx/coverage.txt index 02cfed5b68..f7208d6497 100644 --- a/requirements/edx/coverage.txt +++ b/requirements/edx/coverage.txt @@ -12,7 +12,11 @@ jinja2-pluralize==0.3.0 # via diff-cover jinja2==2.10.3 # via diff-cover, jinja2-pluralize markupsafe==1.1.1 # via jinja2 more-itertools==8.0.2 # via zipp +numpy==1.18.0 # via pandas +pandas==0.24.2 pluggy==0.13.1 # via diff-cover pygments==2.5.2 # via diff-cover +python-dateutil==2.5.3 # via pandas +pytz==2019.3 # via pandas six==1.13.0 # via diff-cover zipp==0.6.0 # via importlib-metadata diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index b56b34048b..0a2e64153f 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -118,7 +118,7 @@ edx-django-release-util==0.3.2 edx-django-sites-extensions==2.4.2 edx-django-utils==2.0.2 edx-drf-extensions==2.4.5 -edx-enterprise==2.0.35 +edx-enterprise==2.0.36 edx-i18n-tools==0.5.0 edx-lint==1.3.0 edx-milestones==0.2.6 @@ -126,13 +126,14 @@ edx-oauth2-provider==1.3.1 edx-opaque-keys[django]==2.0.1 edx-organizations==2.2.0 edx-proctoring-proctortrack==1.0.5 -edx-proctoring==2.2.3 +edx-proctoring==2.2.4 edx-rbac==1.0.5 edx-rest-api-client==1.9.2 edx-search==1.2.2 edx-sga==0.10.0 edx-sphinx-theme==1.5.0 edx-submissions==3.0.3 +edx-tincan-py35==0.0.5 edx-user-state-client==1.1.2 edx-when==0.5.2 edxval==1.1.33 @@ -208,6 +209,7 @@ git+https://github.com/joestump/python-oauth2.git@b94f69b1ad195513547924e380d926 oauthlib==2.1.0 git+https://github.com/edx/edx-ora2.git@2.5.4#egg=ora2==2.5.4 packaging==19.2 +pandas==0.24.2 path.py==8.2.1 pathlib2==2.3.5 pathtools==0.1.2 @@ -249,10 +251,12 @@ pytest-attrib==0.1.3 pytest-cov==2.8.1 pytest-django==3.7.0 pytest-forked==1.1.3 +pytest-json-report==1.2.1 +pytest-metadata==1.8.0 pytest-randomly==3.2.0 pytest-xdist==1.31.0 pytest==5.3.2 -python-dateutil==2.4.0 +python-dateutil==2.5.3 python-levenshtein==0.12.0 python-memcached==1.59 python-slugify==4.0.0 @@ -306,7 +310,6 @@ super-csv==0.9.6 sympy==1.5 testfixtures==6.10.3 text-unidecode==1.3 -tincan==0.0.5 toml==0.10.0 tox-battery==0.5.1 tox==3.14.3 @@ -320,7 +323,7 @@ virtualenv==16.7.9 voluptuous==0.11.7 vulture==1.2 watchdog==0.9.0 -wcwidth==0.1.7 +wcwidth==0.1.8 web-fragments==0.3.1 webencodings==0.5.1 webob==1.8.5 diff --git a/requirements/edx/testing.in b/requirements/edx/testing.in index cf2bb98a9a..2106fdd5ab 100644 --- a/requirements/edx/testing.in +++ b/requirements/edx/testing.in @@ -37,6 +37,7 @@ pytest-attrib # Select tests based on attributes pytest-cov # pytest plugin for measuring code coverage git+https://github.com/nedbat/coverage_pytest_plugin.git@29de030251471e200ff255eb9e549218cd60e872#egg=coverage_pytest_plugin==0.0 pytest-django # Django support for pytest +pytest-json-report # Output json formatted warnings after running pytest pytest-randomly # pytest plugin to randomly order tests pytest-xdist # Parallel execution of tests on multiple CPU cores or hosts radon # Calculates cyclomatic complexity of Python code (code quality utility) diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index e602e10f57..c5531c39f3 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -115,7 +115,7 @@ edx-django-release-util==0.3.2 edx-django-sites-extensions==2.4.2 edx-django-utils==2.0.2 edx-drf-extensions==2.4.5 -edx-enterprise==2.0.35 +edx-enterprise==2.0.36 edx-i18n-tools==0.5.0 edx-lint==1.3.0 edx-milestones==0.2.6 @@ -123,12 +123,13 @@ edx-oauth2-provider==1.3.1 edx-opaque-keys[django]==2.0.1 edx-organizations==2.2.0 edx-proctoring-proctortrack==1.0.5 -edx-proctoring==2.2.3 +edx-proctoring==2.2.4 edx-rbac==1.0.5 edx-rest-api-client==1.9.2 edx-search==1.2.2 edx-sga==0.10.0 edx-submissions==3.0.3 +edx-tincan-py35==0.0.5 edx-user-state-client==1.1.2 edx-when==0.5.2 edxval==1.1.33 @@ -200,6 +201,7 @@ git+https://github.com/joestump/python-oauth2.git@b94f69b1ad195513547924e380d926 oauthlib==2.1.0 git+https://github.com/edx/edx-ora2.git@2.5.4#egg=ora2==2.5.4 packaging==19.2 # via pytest, tox +pandas==0.24.2 path.py==8.2.1 pathlib2==2.3.5 # via pytest pathtools==0.1.2 @@ -238,10 +240,12 @@ pytest-attrib==0.1.3 pytest-cov==2.8.1 pytest-django==3.7.0 pytest-forked==1.1.3 # via pytest-xdist +pytest-json-report==1.2.1 +pytest-metadata==1.8.0 # via pytest-json-report pytest-randomly==3.2.0 pytest-xdist==1.31.0 pytest==5.3.2 -python-dateutil==2.4.0 +python-dateutil==2.5.3 python-levenshtein==0.12.0 python-memcached==1.59 python-slugify==4.0.0 @@ -285,7 +289,6 @@ super-csv==0.9.6 sympy==1.5 testfixtures==6.10.3 text-unidecode==1.3 -tincan==0.0.5 toml==0.10.0 # via tox tox-battery==0.5.1 tox==3.14.3 @@ -298,7 +301,7 @@ user-util==0.1.5 virtualenv==16.7.9 # via tox voluptuous==0.11.7 watchdog==0.9.0 -wcwidth==0.1.7 # via pytest +wcwidth==0.1.8 # via pytest web-fragments==0.3.1 webencodings==0.5.1 webob==1.8.5 diff --git a/scripts/Jenkinsfiles/python b/scripts/Jenkinsfiles/python index 8aa44faa22..eee5a7e48c 100644 --- a/scripts/Jenkinsfiles/python +++ b/scripts/Jenkinsfiles/python @@ -8,18 +8,31 @@ def runPythonTests() { noTags: true, shallow: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker', refspec: git_refspec, url: "git@github.com:edx/${REPO_NAME}.git"]]] sh 'bash scripts/all-tests.sh' - stash includes: 'reports/**/*coverage*', name: "${TEST_SUITE}-reports" + stash includes: 'reports/**/*coverage*,test_root/log/*.json', name: "${TEST_SUITE}-reports" } } def pythonTestCleanup() { - archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/**/*,test_root/log/**/*.log,**/nosetests.xml,*.log' + archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/**/*,test_root/log/**/*.log,test_root/log/*.json,**/nosetests.xml,*.log' sendSplunkFile excludes: '', includes: '**/timing*.log', sizeLimit: '10MB' junit '**/nosetests.xml' sh '''source $HOME/edx-venv-$PYTHON_VERSION/edx-venv/bin/activate bash scripts/xdist/terminate_xdist_nodes.sh''' } +def createWarningsReport(fileExtension){ + println "Creating warnings report for ${fileExtension}" + warning_filename = "warning_report_${fileExtension}.html" + sh """source $HOME/edx-venv-$PYTHON_VERSION/edx-venv/bin/activate + python openedx/core/process_warnings.py --dir-path test_root/log --html-path reports/pytest_warnings/${warning_filename}""" + + publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true, + reportDir: 'reports/pytest_warnings', reportFiles: "${warning_filename}", + reportName: "${warning_filename}", reportTitles: '']) + archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/pytest_warnings/*.html' +} + + def xdist_git_branch() { if (env.ghprbActualCommit) { return "${ghprbActualCommit}" @@ -142,6 +155,8 @@ pipeline { } } } + + stage('Run coverage') { environment { CODE_COV_TOKEN = credentials('CODE_COV_TOKEN') @@ -172,6 +187,7 @@ pipeline { sh """export CI_BRANCH=$ci_branch export TARGET_BRANCH=$target_branch ./scripts/jenkins-report.sh""" + createWarningsReport("all") } } } diff --git a/setup.cfg b/setup.cfg index 35c89aa0a3..5d929ac641 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [tool:pytest] DJANGO_SETTINGS_MODULE = lms.envs.test -addopts = --nomigrations --reuse-db --durations=20 +addopts = --nomigrations --reuse-db --durations=20 --json-report --json-report-omit keywords streams collectors log traceback tests --json-report-file=none # Enable default handling for all warnings, including those that are ignored by default; # but hide rate-limit warnings (because we deliberately don't throttle test user logins) # and field_data deprecation warnings (because fixing them requires a major low-priority refactoring)