Merge pull request #19156 from edx/youngstrom/xdist-test-lsit

Add script to get test list for failed pytest workers

docs/testing.rst

@@ -342,6 +342,14 @@ Note: More on the ``--failed`` functionality:
   failing tests for lms or cms, use the ``paver test_system -s lms --failed``
   or ``paver test_system -s cms --failed`` commands.
 
+Debugging Unittest Flakiness
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+As we move over to running our unittests with Jenkins Pipelines and pytest-xdist,
+there are new ways for tests to flake, which can sometimes be difficult to debug.
+If you run into flakiness, check (and feel free to contribute to) this
+`confluence document <https://openedx.atlassian.net/wiki/spaces/TE/pages/884998163/Debugging+test+failures+with+pytest-xdist>`__ for help.
+
 Running Javascript Unit Tests
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

scripts/xdist/get_worker_test_list.py

@@ -0,0 +1,51 @@
+"""
+This script strips the console log of a pytest-xdist Jenkins run into the test
+lists of each pytest worker.
+Assumes the following format:
+[test-suite] [worker] RESULT test
+"""
+import click
+import io
+import re
+import os
+import shutil
+
+
+@click.command()
+@click.option(
+    '--log-file',
+    help="File name of console log .txt file from a Jenkins build "
+    "that ran pytest-xdist. This can be acquired by running: "
+    "curl -o console.txt https://build.testeng.edx.org/job/JOBNAME/BUILDNUMBER/consoleText",
+    required=True
+)
+@click.option(
+    '--test-suite',
+    help="Test suite that the pytest worker ran. Example: lms-unit",
+    required=True
+)
+def main(log_file, test_suite):
+    worker_test_dict = {}
+    with io.open(log_file, 'r') as console_file:
+        for line in console_file:
+            regex_search = re.search(r'\[{}] \[gw(\d+)] PASSED|FAILED|SKIPPED|ERROR'.format(test_suite), line)
+            if regex_search:
+                worker_num_string = regex_search.group(1)
+                if worker_num_string not in worker_test_dict:
+                    worker_test_dict[worker_num_string] = []
+                worker_test_dict[worker_num_string].append(line.split()[3])
+
+    output_folder_name = "worker_list_files"
+    if os.path.isdir(output_folder_name):
+        shutil.rmtree(output_folder_name)
+    os.mkdir(output_folder_name)
+
+    for worker_num in worker_test_dict:
+        output_file_name = "{}/{}_gw{}_test_list.txt".format(output_folder_name, test_suite, worker_num)
+        with io.open(output_file_name, 'w') as output_file:
+            for line in worker_test_dict[worker_num]:
+                output_file.write(line + "\n")
+
+
+if __name__ == "__main__":
+    main()