test: run openedx and common tests with both lms and cms settings (#29676)

scripts/gha-shards-readme.md

@@ -0,0 +1,35 @@
+# Unit tests sharding strategy
+
+#### background
+Unit tests are run in parallel (in GitHub Actions matrices) using the sharding strategy specified in unit-test-shards.json
+We've divided the top level modules into multiple shards to achieve better parallelism.
+The configuration in unit-test-shards.json specifies the shard name as key for each shard and the value contains an object
+with django settings for each module and paths for submodules to test for example:
+```json
+{
+    "lms-1": {
+        "paths": ["lms/djangoapps/course_api", ...],
+        "settings": "lms.envs.test",
+    }
+    .
+    .
+    .
+}
+```
+The `common` and `openedx` modules are tested with both `lms` and `cms` settings; that's why there are shards with the same `openedx`
+submodules but with different Django settings.
+For more details on sharding strategy please refer to this section on [sharding](https://openedx.atlassian.net/wiki/spaces/AT/pages/3235971586/edx-platfrom+unit+tests+migration+from+Jenkins+to+Github+Actions#Motivation-for-sharding-manually)
+
+#### Unit tests count check is failing
+There's a check in place that makes sure that all the unit tests under edx-platform modules are specified in `unit-test-shards.json`
+If there's a mismatch between the number of unit tests collected from `unit-test-shards.json` and the number of unit tests collected
+against the entire codebase the check will fail.
+You'd have to update the `unit-test-shards.json` file manually to fix this.
+
+##### How to fix
+- If you've added a new django app to the codebase, and you want to add it to the unit tests you need to add it to the `unit-test-shards.json`, details on where (in which shard) to place your Django app please refer to the [sharding](https://openedx.atlassian.net/wiki/spaces/AT/pages/3235971586/edx-platfrom+unit+tests+migration+from+Jenkins+to+Github+Actions#Where-should-I-place-my-new-Django-app) section in this document.
+- If you haven't added any new django app to the codebase, you can debug / verify this by collecting unit tests against a submodule by running `pytest` for example:
+```
+pytest --collect-only --ds=cms.envs.test cms/
+```
+For more details on how this check collects and compares the unit tests count please take a look at [verify unit tests count](../.github/workflows/verify-gha-unit-tests-count.yml)

scripts/gha_unit_tests_collector.py

@@ -1,41 +1,44 @@
-import sys
-import os
-import yaml
 import argparse
 import json
+import sys
 
 
 def get_all_unit_test_shards():
-    unit_tests_json = f'{os.getcwd()}/.github/workflows/unit-test-shards.json'
+    unit_tests_json = '.github/workflows/unit-test-shards.json'
     with open(unit_tests_json) as file:
-        unit_test_workflow_shards = json.loads(file.read())
+        unit_test_workflow_shards = json.load(file)
 
     return unit_test_workflow_shards
 
 
-def get_modules_except_cms():
-    all_unit_test_shards = get_all_unit_test_shards()
-    return [paths for shard_name, paths in all_unit_test_shards.items() if not paths.startswith('cms')]
+def update_unit_test_modules(module_name, shard_config, unit_test_modules):
+    is_cms_shard_path = shard_config['paths'][0].startswith('cms')
+
+    if is_cms_shard_path and module_name == "cms":
+        unit_test_modules.update(shard_config.get('paths'))
+    elif not is_cms_shard_path and module_name != "cms":
+        unit_test_modules.update(shard_config.get('paths'))
+    return unit_test_modules
 
 
-def get_cms_modules():
+def get_unit_test_modules(module_name="lms"):
+    unit_test_modules = set()
     all_unit_test_shards = get_all_unit_test_shards()
-    return [paths for shard_name, paths in all_unit_test_shards.items() if paths.startswith('cms')]
+    for shard_name, shard_config in all_unit_test_shards.items():
+        unit_test_modules = update_unit_test_modules(module_name, shard_config, unit_test_modules)
+    return unit_test_modules
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--cms-only", action="store_true", default="")
     parser.add_argument("--lms-only", action="store_true", default="")
-
     argument = parser.parse_args()
 
-    if argument.lms_only:
-        modules = get_modules_except_cms()
-    elif argument.cms_only:
-        modules = get_cms_modules()
-    else:
-        modules = get_all_unit_test_modules()
+    if not argument.cms_only and not argument.lms_only:
+        print("Please specify --cms-only or --lms-only")
+        sys.exit(1)
 
-    unit_test_paths = ' '.join(modules)
-    sys.stdout.write(unit_test_paths)
+    modules = get_unit_test_modules("cms") if argument.cms_only else get_unit_test_modules("lms")
+    paths_output = ' '.join(modules)
+    print(paths_output)

scripts/unit_test_shards_parser.py

@@ -1,27 +1,39 @@
-import sys
-import os
 import argparse
 import json
+import sys
+
+
+def load_unit_test_shards(shard_name):
+    unit_tests_json = '.github/workflows/unit-test-shards.json'
+    with open(unit_tests_json) as file:
+        unit_test_workflow_shards = json.load(file)
+    if shard_name not in unit_test_workflow_shards:
+        sys.stdout.write("Error, invalid shard name provided. please provide a valid shard name as specified in unit-test-shards.json")
+    return unit_test_workflow_shards
 
 
 def get_test_paths_for_shard(shard_name):
-    unit_tests_json = f'{os.getcwd()}/.github/workflows/unit-test-shards.json'
-    with open(unit_tests_json) as file:
-        unit_test_workflow_shards = json.loads(file.read())
+    return load_unit_test_shards(shard_name).get(shard_name).get("paths")
 
-    if shard_name not in unit_test_workflow_shards:
-        sys.stdout.write("Error, invalid shard name provided. please provide a valid shard name as specified in unit-test-shards.json")
 
-    return unit_test_workflow_shards.get(shard_name)
+def get_settings_for_shard(shard_name):
+    return load_unit_test_shards(shard_name).get(shard_name).get("settings")
+
+
+def get_output(shard_name, output_argument):
+    if output_argument == "settings":
+        return get_settings_for_shard(shard_name)
+    return " ".join(get_test_paths_for_shard(shard_name))
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--shard-name", action="store", default="")
+    parser.add_argument("--output", action="store", default="path", choices=["path", "settings"])
     argument = parser.parse_args()
 
     if not argument.shard_name:
-        sys.stdout.write("Error, no shard name provided. please provide a valid shard name as specified in unit-test-shards.json")
+        sys.exit("Error, no shard name provided. please provide a valid shard name as specified in unit-test-shards.json")
 
-    unit_test_paths = get_test_paths_for_shard(argument.shard_name)
-    sys.stdout.write(unit_test_paths)
+    output = get_output(argument.shard_name, argument.output)
+    print(output)