From 78d9892319937e43149829f6501f59558e43f9a7 Mon Sep 17 00:00:00 2001
From: Michael Youngstrom <youngstrom.m@husky.neu.edu>
Date: Wed, 24 Oct 2018 13:57:48 -0400
Subject: [PATCH] Add script to get test list for failed worker

---
 docs/testing.rst                      |  8 +++++
 scripts/xdist/get_worker_test_list.py | 51 +++++++++++++++++++++++++++
 2 files changed, 59 insertions(+)
 create mode 100644 scripts/xdist/get_worker_test_list.py

diff --git a/docs/testing.rst b/docs/testing.rst
index 9695dcf298..d8e89e417d 100644
--- a/docs/testing.rst
+++ b/docs/testing.rst
@@ -342,6 +342,14 @@ Note: More on the ``--failed`` functionality:
   failing tests for lms or cms, use the ``paver test_system -s lms --failed``
   or ``paver test_system -s cms --failed`` commands.
 
+Debugging Unittest Flakiness
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+As we move over to running our unittests with Jenkins Pipelines and pytest-xdist,
+there are new ways for tests to flake, which can sometimes be difficult to debug.
+If you run into flakiness, check (and feel free to contribute to) this
+`confluence document <https://openedx.atlassian.net/wiki/spaces/TE/pages/884998163/Debugging+test+failures+with+pytest-xdist>`__ for help.
+
 Running Javascript Unit Tests
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/scripts/xdist/get_worker_test_list.py b/scripts/xdist/get_worker_test_list.py
new file mode 100644
index 0000000000..332cc94a6f
--- /dev/null
+++ b/scripts/xdist/get_worker_test_list.py
@@ -0,0 +1,51 @@
+"""
+This script strips the console log of a pytest-xdist Jenkins run into the test
+lists of each pytest worker.
+Assumes the following format:
+[test-suite] [worker] RESULT test
+"""
+import click
+import io
+import re
+import os
+import shutil
+
+
+@click.command()
+@click.option(
+    '--log-file',
+    help="File name of console log .txt file from a Jenkins build "
+    "that ran pytest-xdist. This can be acquired by running: "
+    "curl -o console.txt https://build.testeng.edx.org/job/JOBNAME/BUILDNUMBER/consoleText",
+    required=True
+)
+@click.option(
+    '--test-suite',
+    help="Test suite that the pytest worker ran. Example: lms-unit",
+    required=True
+)
+def main(log_file, test_suite):
+    worker_test_dict = {}
+    with io.open(log_file, 'r') as console_file:
+        for line in console_file:
+            regex_search = re.search(r'\[{}] \[gw(\d+)] PASSED|FAILED|SKIPPED|ERROR'.format(test_suite), line)
+            if regex_search:
+                worker_num_string = regex_search.group(1)
+                if worker_num_string not in worker_test_dict:
+                    worker_test_dict[worker_num_string] = []
+                worker_test_dict[worker_num_string].append(line.split()[3])
+
+    output_folder_name = "worker_list_files"
+    if os.path.isdir(output_folder_name):
+        shutil.rmtree(output_folder_name)
+    os.mkdir(output_folder_name)
+
+    for worker_num in worker_test_dict:
+        output_file_name = "{}/{}_gw{}_test_list.txt".format(output_folder_name, test_suite, worker_num)
+        with io.open(output_file_name, 'w') as output_file:
+            for line in worker_test_dict[worker_num]:
+                output_file.write(line + "\n")
+
+
+if __name__ == "__main__":
+    main()