Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: run openedx and common tests with both lms and cms settings (#29676)

Browse Source
This commit is contained in:
Muhammad Soban Javed
2022-01-21 11:58:57 +05:00
committed by GitHub
parent 003bbe3482
commit 4e22a38ca5
6 changed files with 361 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

285
.github/workflows/unit-test-shards.json vendored
View File

@@ -1,14 +1,275 @@
{
"lms-1": "lms/djangoapps/badges/ lms/djangoapps/branding/ lms/djangoapps/bulk_email/ lms/djangoapps/bulk_enroll/ lms/djangoapps/bulk_user_retirement/ lms/djangoapps/ccx/ lms/djangoapps/certificates/ lms/djangoapps/commerce/",
"lms-2": "lms/djangoapps/course_api/ lms/djangoapps/course_blocks/ lms/djangoapps/course_goals/ lms/djangoapps/course_home_api/ lms/djangoapps/course_wiki/ lms/djangoapps/coursewarehistoryextended/ lms/djangoapps/debug/",
"lms-3": "lms/djangoapps/courseware/",
"lms-4": "lms/djangoapps/discussion/ lms/djangoapps/edxnotes/ lms/djangoapps/email_marketing/ lms/djangoapps/experiments/",
"lms-5": "lms/djangoapps/gating/ lms/djangoapps/grades/ lms/djangoapps/instructor/ lms/djangoapps/instructor_analytics/",
"lms-6": "lms/djangoapps/instructor_task/ lms/djangoapps/learner_dashboard/ lms/djangoapps/lms_initialization/ lms/djangoapps/lms_xblock/ lms/djangoapps/lti_provider/ lms/djangoapps/mailing/ lms/djangoapps/mobile_api/ lms/djangoapps/monitoring/ lms/djangoapps/program_enrollments/ lms/djangoapps/rss_proxy/ lms/djangoapps/save_for_later/ lms/djangoapps/static_template_view/ lms/djangoapps/staticbook/ lms/djangoapps/support/ lms/djangoapps/survey/ lms/djangoapps/teams/ lms/djangoapps/tests/ lms/djangoapps/user_tours/ lms/djangoapps/verify_student/ lms/envs/ lms/lib/ lms/tests.py",
"openedx-1": "openedx/core/djangoapps/ace_common/ openedx/core/djangoapps/cors_csrf/ openedx/core/djangoapps/agreements/ openedx/core/djangoapps/api_admin/ openedx/core/djangoapps/auth_exchange/ openedx/core/djangoapps/bookmarks/ openedx/core/djangoapps/cache_toolbox/ openedx/core/djangoapps/catalog/ openedx/core/djangoapps/ccxcon/ openedx/core/djangoapps/commerce/ openedx/core/djangoapps/common_initialization/ openedx/core/djangoapps/common_views/ openedx/core/djangoapps/config_model_utils/ openedx/core/djangoapps/content/ openedx/core/djangoapps/content_libraries/ openedx/core/djangoapps/contentserver/ openedx/core/djangoapps/cookie_metadata/ openedx/core/djangoapps/course_apps/ openedx/core/djangoapps/course_date_signals/ openedx/core/djangoapps/course_groups/ openedx/core/djangoapps/coursegraph/ openedx/core/djangoapps/courseware_api/ openedx/core/djangoapps/crawlers/ openedx/core/djangoapps/credentials/ openedx/core/djangoapps/credit/ openedx/core/djangoapps/dark_lang/ openedx/core/djangoapps/debug/ openedx/core/djangoapps/demographics/ openedx/core/djangoapps/discussions/ openedx/core/djangoapps/django_comment_common/ openedx/core/djangoapps/embargo/ openedx/core/djangoapps/enrollments/ openedx/core/djangoapps/external_user_ids/",
"openedx-2": "openedx/core/djangoapps/geoinfo/ openedx/core/djangoapps/header_control/ openedx/core/djangoapps/heartbeat/ openedx/core/djangoapps/lang_pref/ openedx/core/djangoapps/models/ openedx/core/djangoapps/monkey_patch/ openedx/core/djangoapps/oauth_dispatch/ openedx/core/djangoapps/olx_rest_api/ openedx/core/djangoapps/password_policy/ openedx/core/djangoapps/plugin_api/ openedx/core/djangoapps/plugins/ openedx/core/djangoapps/profile_images/ openedx/core/djangoapps/programs/ openedx/core/djangoapps/safe_sessions/ openedx/core/djangoapps/schedules/ openedx/core/djangoapps/self_paced/ openedx/core/djangoapps/service_status/ openedx/core/djangoapps/session_inactivity_timeout/ openedx/core/djangoapps/signals/ openedx/core/djangoapps/site_configuration/ openedx/core/djangoapps/system_wide_roles/ openedx/core/djangoapps/theming/ openedx/core/djangoapps/user_api/ openedx/core/djangoapps/user_authn/ openedx/core/djangoapps/util/ openedx/core/djangoapps/verified_track_content/ openedx/core/djangoapps/video_config/ openedx/core/djangoapps/video_pipeline/ openedx/core/djangoapps/waffle_utils/ openedx/core/djangoapps/xblock/ openedx/core/djangoapps/xmodule_django/ openedx/core/djangoapps/zendesk_proxy/ openedx/core/djangolib/ openedx/core/lib/ openedx/core/tests/ openedx/core/tests/ openedx/features/ openedx/testing/ openedx/tests/",
"cms-1": "cms/djangoapps/api/ cms/djangoapps/cms_user_tasks/ cms/djangoapps/course_creators/ cms/djangoapps/export_course_metadata/ cms/djangoapps/maintenance/ cms/djangoapps/models/ cms/djangoapps/pipeline_js/ cms/djangoapps/xblock_config/ cms/envs/ cms/lib/",
"cms-2": "cms/djangoapps/contentstore/",
"common-1": "common/djangoapps/",
"common-2": "common/lib/"
"lms-1": {
"settings": "lms.envs.test",
"paths": [
"lms/djangoapps/badges/",
"lms/djangoapps/branding/",
"lms/djangoapps/bulk_email/",
"lms/djangoapps/bulk_enroll/",
"lms/djangoapps/bulk_user_retirement/",
"lms/djangoapps/ccx/",
"lms/djangoapps/certificates/",
"lms/djangoapps/commerce/"
]
},
"lms-2": {
"settings": "lms.envs.test",
"paths": [
"lms/djangoapps/course_api/",
"lms/djangoapps/course_blocks/",
"lms/djangoapps/course_goals/",
"lms/djangoapps/course_home_api/",
"lms/djangoapps/course_wiki/",
"lms/djangoapps/coursewarehistoryextended/",
"lms/djangoapps/debug/"
]
},
"lms-3": {
"settings": "lms.envs.test",
"paths": [
"lms/djangoapps/courseware/"
]
},
"lms-4": {
"settings": "lms.envs.test",
"paths": [
"lms/djangoapps/discussion/",
"lms/djangoapps/edxnotes/",
"lms/djangoapps/email_marketing/",
"lms/djangoapps/experiments/"
]
},
"lms-5": {
"settings": "lms.envs.test",
"paths": [
"lms/djangoapps/gating/",
"lms/djangoapps/grades/",
"lms/djangoapps/instructor/",
"lms/djangoapps/instructor_analytics/"
]
},
"lms-6": {
"settings": "lms.envs.test",
"paths": [
"lms/djangoapps/instructor_task/",
"lms/djangoapps/learner_dashboard/",
"lms/djangoapps/lms_initialization/",
"lms/djangoapps/lms_xblock/",
"lms/djangoapps/lti_provider/",
"lms/djangoapps/mailing/",
"lms/djangoapps/mobile_api/",
"lms/djangoapps/monitoring/",
"lms/djangoapps/program_enrollments/",
"lms/djangoapps/rss_proxy/",
"lms/djangoapps/save_for_later/",
"lms/djangoapps/static_template_view/",
"lms/djangoapps/staticbook/",
"lms/djangoapps/support/",
"lms/djangoapps/survey/",
"lms/djangoapps/teams/",
"lms/djangoapps/tests/",
"lms/djangoapps/user_tours/",
"lms/djangoapps/verify_student/",
"lms/envs/",
"lms/lib/",
"lms/tests.py"
]
},
"openedx-1": {
"settings": "lms.envs.test",
"paths": [
"openedx/core/djangoapps/ace_common/",
"openedx/core/djangoapps/cors_csrf/",
"openedx/core/djangoapps/agreements/",
"openedx/core/djangoapps/api_admin/",
"openedx/core/djangoapps/auth_exchange/",
"openedx/core/djangoapps/bookmarks/",
"openedx/core/djangoapps/cache_toolbox/",
"openedx/core/djangoapps/catalog/",
"openedx/core/djangoapps/ccxcon/",
"openedx/core/djangoapps/commerce/",
"openedx/core/djangoapps/common_initialization/",
"openedx/core/djangoapps/common_views/",
"openedx/core/djangoapps/config_model_utils/",
"openedx/core/djangoapps/content/",
"openedx/core/djangoapps/content_libraries/",
"openedx/core/djangoapps/contentserver/",
"openedx/core/djangoapps/cookie_metadata/",
"openedx/core/djangoapps/course_apps/",
"openedx/core/djangoapps/course_date_signals/",
"openedx/core/djangoapps/course_groups/",
"openedx/core/djangoapps/coursegraph/",
"openedx/core/djangoapps/courseware_api/",
"openedx/core/djangoapps/crawlers/",
"openedx/core/djangoapps/credentials/",
"openedx/core/djangoapps/credit/",
"openedx/core/djangoapps/dark_lang/",
"openedx/core/djangoapps/debug/",
"openedx/core/djangoapps/demographics/",
"openedx/core/djangoapps/discussions/",
"openedx/core/djangoapps/django_comment_common/",
"openedx/core/djangoapps/embargo/",
"openedx/core/djangoapps/enrollments/",
"openedx/core/djangoapps/external_user_ids/"
]
},
"openedx-2": {
"settings": "lms.envs.test",
"paths": [
"openedx/core/djangoapps/geoinfo/",
"openedx/core/djangoapps/header_control/",
"openedx/core/djangoapps/heartbeat/",
"openedx/core/djangoapps/lang_pref/",
"openedx/core/djangoapps/models/",
"openedx/core/djangoapps/monkey_patch/",
"openedx/core/djangoapps/oauth_dispatch/",
"openedx/core/djangoapps/olx_rest_api/",
"openedx/core/djangoapps/password_policy/",
"openedx/core/djangoapps/plugin_api/",
"openedx/core/djangoapps/plugins/",
"openedx/core/djangoapps/profile_images/",
"openedx/core/djangoapps/programs/",
"openedx/core/djangoapps/safe_sessions/",
"openedx/core/djangoapps/schedules/",
"openedx/core/djangoapps/self_paced/",
"openedx/core/djangoapps/service_status/",
"openedx/core/djangoapps/session_inactivity_timeout/",
"openedx/core/djangoapps/signals/",
"openedx/core/djangoapps/site_configuration/",
"openedx/core/djangoapps/system_wide_roles/",
"openedx/core/djangoapps/theming/",
"openedx/core/djangoapps/user_api/",
"openedx/core/djangoapps/user_authn/",
"openedx/core/djangoapps/util/",
"openedx/core/djangoapps/verified_track_content/",
"openedx/core/djangoapps/video_config/",
"openedx/core/djangoapps/video_pipeline/",
"openedx/core/djangoapps/waffle_utils/",
"openedx/core/djangoapps/xblock/",
"openedx/core/djangoapps/xmodule_django/",
"openedx/core/djangoapps/zendesk_proxy/",
"openedx/core/djangolib/",
"openedx/core/lib/",
"openedx/core/tests/",
"openedx/features/",
"openedx/testing/",
"openedx/tests/"
]
},
"openedx-3": {
"settings": "cms.envs.test",
"paths": [
"openedx/core/djangoapps/ace_common/",
"openedx/core/djangoapps/cors_csrf/",
"openedx/core/djangoapps/agreements/",
"openedx/core/djangoapps/api_admin/",
"openedx/core/djangoapps/auth_exchange/",
"openedx/core/djangoapps/bookmarks/",
"openedx/core/djangoapps/cache_toolbox/",
"openedx/core/djangoapps/catalog/",
"openedx/core/djangoapps/ccxcon/",
"openedx/core/djangoapps/commerce/",
"openedx/core/djangoapps/common_initialization/",
"openedx/core/djangoapps/common_views/",
"openedx/core/djangoapps/config_model_utils/",
"openedx/core/djangoapps/content/",
"openedx/core/djangoapps/content_libraries/",
"openedx/core/djangoapps/contentserver/",
"openedx/core/djangoapps/cookie_metadata/",
"openedx/core/djangoapps/course_apps/",
"openedx/core/djangoapps/course_date_signals/",
"openedx/core/djangoapps/course_groups/",
"openedx/core/djangoapps/coursegraph/",
"openedx/core/djangoapps/courseware_api/",
"openedx/core/djangoapps/crawlers/",
"openedx/core/djangoapps/credentials/",
"openedx/core/djangoapps/credit/",
"openedx/core/djangoapps/dark_lang/",
"openedx/core/djangoapps/debug/",
"openedx/core/djangoapps/demographics/",
"openedx/core/djangoapps/discussions/",
"openedx/core/djangoapps/django_comment_common/",
"openedx/core/djangoapps/embargo/",
"openedx/core/djangoapps/enrollments/",
"openedx/core/djangoapps/external_user_ids/"
]
},
"openedx-4": {
"settings": "cms.envs.test",
"paths": [
"openedx/core/djangoapps/geoinfo/",
"openedx/core/djangoapps/header_control/",
"openedx/core/djangoapps/heartbeat/",
"openedx/core/djangoapps/lang_pref/",
"openedx/core/djangoapps/models/",
"openedx/core/djangoapps/monkey_patch/",
"openedx/core/djangoapps/oauth_dispatch/",
"openedx/core/djangoapps/olx_rest_api/",
"openedx/core/djangoapps/password_policy/",
"openedx/core/djangoapps/plugin_api/",
"openedx/core/djangoapps/plugins/",
"openedx/core/djangoapps/profile_images/",
"openedx/core/djangoapps/programs/",
"openedx/core/djangoapps/safe_sessions/",
"openedx/core/djangoapps/schedules/",
"openedx/core/djangoapps/self_paced/",
"openedx/core/djangoapps/service_status/",
"openedx/core/djangoapps/session_inactivity_timeout/",
"openedx/core/djangoapps/signals/",
"openedx/core/djangoapps/site_configuration/",
"openedx/core/djangoapps/system_wide_roles/",
"openedx/core/djangoapps/theming/",
"openedx/core/djangoapps/user_api/",
"openedx/core/djangoapps/user_authn/",
"openedx/core/djangoapps/util/",
"openedx/core/djangoapps/verified_track_content/",
"openedx/core/djangoapps/video_config/",
"openedx/core/djangoapps/video_pipeline/",
"openedx/core/djangoapps/waffle_utils/",
"openedx/core/djangoapps/xblock/",
"openedx/core/djangoapps/xmodule_django/",
"openedx/core/djangoapps/zendesk_proxy/",
"openedx/core/lib/",
"openedx/tests/"
]
},
"cms-1": {
"settings": "cms.envs.test",
"paths": [
"cms/djangoapps/api/",
"cms/djangoapps/cms_user_tasks/",
"cms/djangoapps/course_creators/",
"cms/djangoapps/export_course_metadata/",
"cms/djangoapps/maintenance/",
"cms/djangoapps/models/",
"cms/djangoapps/pipeline_js/",
"cms/djangoapps/xblock_config/",
"cms/envs/",
"cms/lib/"
]
},
"cms-2": {
"settings": "cms.envs.test",
"paths": [
"cms/djangoapps/contentstore/"
]
},
"common-1": {
"settings": "lms.envs.test",
"paths": [
"common/djangoapps/"
]
},
"common-2": {
"settings": "lms.envs.test",
"paths": [
"common/lib/"
]
},
"common-3": {
"settings": "cms.envs.test",
"paths": [
"common/djangoapps/"
]
}
}

9
.github/workflows/unit-tests.yml vendored
View File

@@ -22,10 +22,13 @@ jobs:
"lms-6",
"openedx-1",
"openedx-2",
"openedx-3",
"openedx-4",
"cms-1",
"cms-2",
"common-1",
"common-2",
"common-3",
]
@@ -38,13 +41,9 @@ jobs:
run: |
sudo /etc/init.d/mongodb start
- name: set top-level module name
run: |
echo "module_name=$(echo '${{ matrix.shard_name }}' | awk -F '-' '{print $1}')" >> $GITHUB_ENV
- name: set settings path
run: |
echo "settings_path=$(if [ '${{ env.module_name }}' = 'cms' ]; then echo 'cms.envs.test'; else echo 'lms.envs.test' ; fi)" >> $GITHUB_ENV
echo "settings_path=$(python scripts/unit_test_shards_parser.py --shard-name=${{ matrix.shard_name }} --output settings )" >> $GITHUB_ENV
# - name: set pytest randomly option
# run: |

6
.github/workflows/verify-gha-unit-tests-count.yml vendored
View File

@@ -50,8 +50,10 @@ jobs:
echo All root unit tests count: ${{ env.root_all_unit_tests_count }}
echo All shards unit tests count: ${{ env.shards_all_unit_tests_count }}
- name: verify unit tests count
- name: fail the check
if: ${{ env.root_all_unit_tests_count != env.shards_all_unit_tests_count }}
run: |
echo "::error title='Unit test modules in unit-test-shards.json (unit-tests.yml workflow) are outdated'::unit tests running in unit-tests workflow don't match the count for unit tests for entire edx-platform suite, please update the unit-test-shards.json under .github/workflows to add any missing apps and match the count"
echo "::error title='Unit test modules in unit-test-shards.json (unit-tests.yml workflow) are outdated'::unit tests running in unit-tests
workflow don't match the count for unit tests for entire edx-platform suite, please update the unit-test-shards.json under .github/workflows
to add any missing apps and match the count. for more details please take a look at scripts/gha-shards-readme.md"
exit 1

35
scripts/gha-shards-readme.md Normal file
View File

@@ -0,0 +1,35 @@
# Unit tests sharding strategy
#### background
Unit tests are run in parallel (in GitHub Actions matrices) using the sharding strategy specified in unit-test-shards.json
We've divided the top level modules into multiple shards to achieve better parallelism.
The configuration in unit-test-shards.json specifies the shard name as key for each shard and the value contains an object
with django settings for each module and paths for submodules to test for example:
```json
{
"lms-1": {
"paths": ["lms/djangoapps/course_api", ...],
"settings": "lms.envs.test",
}
.
.
.
}
```
The `common` and `openedx` modules are tested with both `lms` and `cms` settings; that's why there are shards with the same `openedx`
submodules but with different Django settings.
For more details on sharding strategy please refer to this section on [sharding](https://openedx.atlassian.net/wiki/spaces/AT/pages/3235971586/edx-platfrom+unit+tests+migration+from+Jenkins+to+Github+Actions#Motivation-for-sharding-manually)
#### Unit tests count check is failing
There's a check in place that makes sure that all the unit tests under edx-platform modules are specified in `unit-test-shards.json`
If there's a mismatch between the number of unit tests collected from `unit-test-shards.json` and the number of unit tests collected
against the entire codebase the check will fail.
You'd have to update the `unit-test-shards.json` file manually to fix this.
##### How to fix
- If you've added a new django app to the codebase, and you want to add it to the unit tests you need to add it to the `unit-test-shards.json`, details on where (in which shard) to place your Django app please refer to the [sharding](https://openedx.atlassian.net/wiki/spaces/AT/pages/3235971586/edx-platfrom+unit+tests+migration+from+Jenkins+to+Github+Actions#Where-should-I-place-my-new-Django-app) section in this document.
- If you haven't added any new django app to the codebase, you can debug / verify this by collecting unit tests against a submodule by running `pytest` for example:
```
pytest --collect-only --ds=cms.envs.test cms/
```
For more details on how this check collects and compares the unit tests count please take a look at [verify unit tests count](../.github/workflows/verify-gha-unit-tests-count.yml)

41
scripts/gha_unit_tests_collector.py
View File

@@ -1,41 +1,44 @@
import sys
import os
import yaml
import argparse
import json
import sys
def get_all_unit_test_shards():
unit_tests_json = f'{os.getcwd()}/.github/workflows/unit-test-shards.json'
unit_tests_json = '.github/workflows/unit-test-shards.json'
with open(unit_tests_json) as file:
unit_test_workflow_shards = json.loads(file.read())
unit_test_workflow_shards = json.load(file)
return unit_test_workflow_shards
def get_modules_except_cms():
all_unit_test_shards = get_all_unit_test_shards()
return [paths for shard_name, paths in all_unit_test_shards.items() if not paths.startswith('cms')]
def update_unit_test_modules(module_name, shard_config, unit_test_modules):
is_cms_shard_path = shard_config['paths'][0].startswith('cms')
if is_cms_shard_path and module_name == "cms":
unit_test_modules.update(shard_config.get('paths'))
elif not is_cms_shard_path and module_name != "cms":
unit_test_modules.update(shard_config.get('paths'))
return unit_test_modules
def get_cms_modules():
def get_unit_test_modules(module_name="lms"):
unit_test_modules = set()
all_unit_test_shards = get_all_unit_test_shards()
return [paths for shard_name, paths in all_unit_test_shards.items() if paths.startswith('cms')]
for shard_name, shard_config in all_unit_test_shards.items():
unit_test_modules = update_unit_test_modules(module_name, shard_config, unit_test_modules)
return unit_test_modules
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--cms-only", action="store_true", default="")
parser.add_argument("--lms-only", action="store_true", default="")
argument = parser.parse_args()
if argument.lms_only:
modules = get_modules_except_cms()
elif argument.cms_only:
modules = get_cms_modules()
else:
modules = get_all_unit_test_modules()
if not argument.cms_only and not argument.lms_only:
print("Please specify --cms-only or --lms-only")
sys.exit(1)
unit_test_paths = ' '.join(modules)
sys.stdout.write(unit_test_paths)
modules = get_unit_test_modules("cms") if argument.cms_only else get_unit_test_modules("lms")
paths_output = ' '.join(modules)
print(paths_output)

34
scripts/unit_test_shards_parser.py
View File

@@ -1,27 +1,39 @@
import sys
import os
import argparse
import json
import sys
def load_unit_test_shards(shard_name):
unit_tests_json = '.github/workflows/unit-test-shards.json'
with open(unit_tests_json) as file:
unit_test_workflow_shards = json.load(file)
if shard_name not in unit_test_workflow_shards:
sys.stdout.write("Error, invalid shard name provided. please provide a valid shard name as specified in unit-test-shards.json")
return unit_test_workflow_shards
def get_test_paths_for_shard(shard_name):
unit_tests_json = f'{os.getcwd()}/.github/workflows/unit-test-shards.json'
with open(unit_tests_json) as file:
unit_test_workflow_shards = json.loads(file.read())
return load_unit_test_shards(shard_name).get(shard_name).get("paths")
if shard_name not in unit_test_workflow_shards:
sys.stdout.write("Error, invalid shard name provided. please provide a valid shard name as specified in unit-test-shards.json")
return unit_test_workflow_shards.get(shard_name)
def get_settings_for_shard(shard_name):
return load_unit_test_shards(shard_name).get(shard_name).get("settings")
def get_output(shard_name, output_argument):
if output_argument == "settings":
return get_settings_for_shard(shard_name)
return " ".join(get_test_paths_for_shard(shard_name))
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--shard-name", action="store", default="")
parser.add_argument("--output", action="store", default="path", choices=["path", "settings"])
argument = parser.parse_args()
if not argument.shard_name:
sys.stdout.write("Error, no shard name provided. please provide a valid shard name as specified in unit-test-shards.json")
sys.exit("Error, no shard name provided. please provide a valid shard name as specified in unit-test-shards.json")
unit_test_paths = get_test_paths_for_shard(argument.shard_name)
sys.stdout.write(unit_test_paths)
output = get_output(argument.shard_name, argument.output)
print(output)