224 lines
7.7 KiB
Python
224 lines
7.7 KiB
Python
"""
|
|
This script can be used to find the fewest number of tests required to get a
|
|
failure, in cases where a test failure is dependent on test order.
|
|
|
|
The script performs the following:
|
|
1. It strips the console log of a pytest-xdist Jenkins run into the test
|
|
lists of each pytest worker until it finds the first failure.
|
|
2. It makes sure that running the single failing test doesn't fail on its
|
|
own.
|
|
3. It then finds the fewest number of tests required to continue to see the
|
|
failure, and outputs the pytest command needed to replicate.
|
|
|
|
Sample usage::
|
|
|
|
python scripts/xdist/find_dependent_test_failures.py --log-file console.txt --test-suite lms-unit
|
|
|
|
"""
|
|
|
|
|
|
import io
|
|
import os
|
|
import re
|
|
import shutil
|
|
import tempfile
|
|
|
|
import click
|
|
|
|
OUTPUT_FOLDER_NAME = "worker_list_files"
|
|
test_suite_option = None
|
|
fast_option = None
|
|
verbose_option = None
|
|
|
|
|
|
@click.command()
|
|
@click.option(
|
|
'--log-file',
|
|
help="File name of console log .txt file from a Jenkins build "
|
|
"that ran pytest-xdist. This can be acquired by running: "
|
|
"curl -o console.txt https://build.testeng.edx.org/job/JOBNAME/BUILDNUMBER/consoleText",
|
|
required=True
|
|
)
|
|
@click.option(
|
|
'--test-suite',
|
|
help="Test suite that the pytest worker ran.",
|
|
type=click.Choice(['lms-unit', 'cms-unit', 'commonlib-unit']),
|
|
required=True
|
|
)
|
|
@click.option(
|
|
'--fast/--slow',
|
|
help="Fast looks for issues in setup/teardown by running one test per class or file.",
|
|
default=True
|
|
)
|
|
@click.option(
|
|
'--verbose/--quiet',
|
|
help="Verbose includes the test output.",
|
|
default=None
|
|
)
|
|
def main(log_file, test_suite, fast, verbose):
|
|
global test_suite_option
|
|
global fast_option
|
|
global verbose_option
|
|
test_suite_option = test_suite
|
|
fast_option = fast
|
|
verbose_option = verbose
|
|
|
|
_clean_output_folder()
|
|
|
|
failing_test_list = _strip_console_for_tests_with_failure(log_file, test_suite)
|
|
|
|
if not failing_test_list:
|
|
print('No failures found in log file.')
|
|
return
|
|
|
|
if _create_and_check_test_files_for_failures(failing_test_list[-1:], 'SINGLE'):
|
|
print("Single test failed. Failures not dependent on order.")
|
|
return
|
|
|
|
test_list_with_failures, pytest_command = _find_fewest_tests_with_failures(failing_test_list, 'ALL')
|
|
if test_list_with_failures:
|
|
print('Found failures running {} tests.'.format(len(test_list_with_failures)))
|
|
print(f'Use: {pytest_command}')
|
|
return
|
|
|
|
if fast_option:
|
|
print('No tests failed locally with --fast option. Try running again with --slow to include more tests.')
|
|
return
|
|
|
|
print('No tests failed locally.')
|
|
|
|
|
|
def _clean_output_folder():
|
|
if os.path.isdir(OUTPUT_FOLDER_NAME):
|
|
shutil.rmtree(OUTPUT_FOLDER_NAME)
|
|
os.mkdir(OUTPUT_FOLDER_NAME)
|
|
|
|
|
|
def _strip_console_for_tests_with_failure(log_file, test_suite):
|
|
"""
|
|
Returns list of tests ending with a failing test, or None if no failures found.
|
|
"""
|
|
global fast_option
|
|
worker_test_dict = {}
|
|
test_base_included = {}
|
|
failing_worker_num = None
|
|
with open(log_file, 'r') as console_file:
|
|
for line in console_file:
|
|
regex_search = re.search(fr'\[gw(\d+)] (PASSED|FAILED|SKIPPED|ERROR) (\S+)', line)
|
|
if regex_search:
|
|
worker_num_string = regex_search.group(1)
|
|
pass_fail_string = regex_search.group(2)
|
|
if worker_num_string not in worker_test_dict:
|
|
worker_test_dict[worker_num_string] = []
|
|
test = regex_search.group(3)
|
|
if test_suite == "commonlib-unit":
|
|
if "pavelib" not in test and not test.startswith('scripts'):
|
|
test = f"common/lib/{test}"
|
|
if fast_option and pass_fail_string == 'PASSED':
|
|
# fast option will only take one test per class or module, in case
|
|
# the failure is a setup/teardown failure.
|
|
test_base = '::'.join(test.split('::')[:-1])
|
|
if test_base not in test_base_included:
|
|
worker_test_dict[worker_num_string].append(test)
|
|
test_base_included[test_base] = True
|
|
elif (not fast_option or (fast_option and pass_fail_string == 'FAILED')):
|
|
worker_test_dict[worker_num_string].append(test)
|
|
if pass_fail_string == 'FAILED':
|
|
failing_worker_num = worker_num_string
|
|
break
|
|
if failing_worker_num:
|
|
return worker_test_dict[failing_worker_num]
|
|
|
|
|
|
def _get_pytest_command(output_file_name):
|
|
"""
|
|
Return the pytest command to run.
|
|
"""
|
|
return f"pytest -p 'no:randomly' `cat {output_file_name}`"
|
|
|
|
|
|
def _run_tests_and_check_for_failures(output_file_name):
|
|
"""
|
|
Runs tests and returns True if failures are found.
|
|
"""
|
|
global verbose_option
|
|
pytest_command = _get_pytest_command(output_file_name)
|
|
test_output = os.popen(pytest_command).read()
|
|
if verbose_option:
|
|
print(test_output)
|
|
failures_search = re.search(r'=== (\d+) failed', test_output)
|
|
return bool(failures_search) and int(failures_search.group(1)) > 0
|
|
|
|
|
|
def _create_and_check_test_files_for_failures(test_list, test_type):
|
|
"""
|
|
Run the test list to see if there are any failures.
|
|
|
|
Keeps around any test files that produced a failure, and deletes
|
|
the passing files.
|
|
|
|
Returns the pytest command to run if failures are found.
|
|
"""
|
|
print("Testing {}, includes {} test(s)...".format(test_type, len(test_list)))
|
|
global test_suite_option
|
|
output_file_name = "{}_failing_test_list_{}_{}.txt".format(
|
|
OUTPUT_FOLDER_NAME, test_suite_option, test_type, len(test_list)
|
|
)
|
|
output_file_path = os.path.join(OUTPUT_FOLDER_NAME, output_file_name)
|
|
# Note: We don't really need a temporary file, and could just output the tests directly
|
|
# to the command line, but this keeps the verbose output cleaner.
|
|
temp_file = tempfile.NamedTemporaryFile(prefix=output_file_name, dir=OUTPUT_FOLDER_NAME, delete=False)
|
|
|
|
with open(temp_file.name, 'w') as output_file:
|
|
for line in test_list:
|
|
output_file.write(line + "\n")
|
|
temp_file.close()
|
|
|
|
if _run_tests_and_check_for_failures(temp_file.name):
|
|
os.rename(temp_file.name, output_file_path)
|
|
print('- test failures found.')
|
|
return _get_pytest_command(output_file_path)
|
|
|
|
os.remove(temp_file.name)
|
|
print('- no failures found.')
|
|
return None
|
|
|
|
|
|
def _find_fewest_tests_with_failures(test_list, test_type):
|
|
"""
|
|
Recursively tests half the tests, finding the smallest number of tests to obtain a failure.
|
|
|
|
Returns:
|
|
(test_list, pytest_command): Tuple with the smallest test_list and the pytest_command
|
|
to be used for testing. Returns (None, None) if no failures are found.
|
|
"""
|
|
if len(test_list) <= 1:
|
|
return None, None
|
|
|
|
pytest_command = _create_and_check_test_files_for_failures(test_list, test_type)
|
|
if not pytest_command:
|
|
return None, None
|
|
|
|
if len(test_list) == 2:
|
|
return test_list, pytest_command
|
|
|
|
half_tests_num = round((len(test_list) - 1) / 2)
|
|
failing_test = test_list[-1:]
|
|
test_list_a = test_list[0:half_tests_num] + failing_test
|
|
test_list_b = test_list[half_tests_num:]
|
|
failing_test_list_a, pytest_command_a = _find_fewest_tests_with_failures(test_list_a, 'GROUP-A')
|
|
if failing_test_list_a:
|
|
return failing_test_list_a, pytest_command_a
|
|
|
|
failing_test_list_b, pytest_command_b = _find_fewest_tests_with_failures(test_list_b, 'GROUP-B')
|
|
if failing_test_list_b:
|
|
return failing_test_list_b, pytest_command_b
|
|
|
|
# This could occur if there is a complex set of dependencies where the
|
|
# original list fails, but neither of its halves (A or B) fail.
|
|
return test_list, pytest_command
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|