diff --git a/lms/djangoapps/monitoring/scripts/clean_unmapped_view_modules.py b/lms/djangoapps/monitoring/scripts/clean_unmapped_view_modules.py new file mode 100644 index 0000000000..2d6233ff5b --- /dev/null +++ b/lms/djangoapps/monitoring/scripts/clean_unmapped_view_modules.py @@ -0,0 +1,73 @@ +""" +Provided a CSV of data from our monitoring system, this script outputs a unique and clean +set of apps of unmapped view_func_modules. + +Context: This script was useful when first introducing ownership mapping and we had many +apps from 3rd-party dependencies that were missed. At this point, we'd probably only +expect 0-2 new unmapped apps, which could be cleaned manually very quickly without this +script. + +Sample usage:: + + python lms/djangoapps/monitoring/scripts/clean_unmapped_view_modules.py --unmapped-csv "unmapped-apps.csv" + +Or for more details:: + + python lms/djangoapps/monitoring/scripts/clean_unmapped_view_modules.py --help + + +""" +import csv +import click + + +@click.command() +@click.option( + '--unmapped-csv', + help="File name of .csv file with unmapped edx-platform view modules.", + required=True +) +def main(unmapped_csv): + """ + Reads CSV of unmapped view_func_modules and outputs a clean list of apps to map. + + NewRelic Insights Query to create CSV of unmapped modules: + + \b + SELECT count(view_func_module) FROM Transaction + WHERE code_owner is null FACET view_func_module + SINCE 1 week ago + LIMIT 50 + + \b + * Increase or decrease SINCE clause as necessary based on when the mappings were last updated. + * Save results as CSV for use in script + + Sample CSV input:: + + \b + View Func Module,View Func Modules + enterprise.api.v1.views,1542 + edx_proctoring.views,116 + social_django.views,53 + + Script removes duplicates in addition to providing sorted list of plain app names. + + """ + with open(unmapped_csv, 'r') as file: + csv_data = file.read() + reader = csv.DictReader(csv_data.splitlines()) + + clean_apps_set = set() + for row in reader: + path = row.get('View Func Module') + path_parts = path.split('.') + clean_apps_set.add(path_parts[0]) + + print('# Move into generate_code_owner_mappings.py and complete mappings.') + for clean_app in sorted(clean_apps_set): + print(clean_app) + + +if __name__ == "__main__": + main() # pylint: disable=no-value-for-parameter diff --git a/lms/djangoapps/monitoring/scripts/generate_code_owner_mappings.py b/lms/djangoapps/monitoring/scripts/generate_code_owner_mappings.py index 99046f5e18..0913e553df 100644 --- a/lms/djangoapps/monitoring/scripts/generate_code_owner_mappings.py +++ b/lms/djangoapps/monitoring/scripts/generate_code_owner_mappings.py @@ -3,61 +3,150 @@ This script generates code owner mappings for monitoring LMS. Sample usage:: - python lms/djangoapps/monitoring/scripts/generate_code_owner_mappings.py --app-csv "edx-platform Apps Ownership.csv" + python lms/djangoapps/monitoring/scripts/generate_code_owner_mappings.py --repo-csv "Individual Repo Ownership.csv" --app-csv "edx-platform Apps Ownership.csv" --dep-csv "edx-platform 3rd-party Ownership.csv" -Sample CSV input:: +Or for more details:: - Path,owner.squad - ./common/djangoapps/xblock_django,team-red - ./openedx/core/djangoapps/xblock,team-red - ./lms/djangoapps/badges,team-blue + python lms/djangoapps/monitoring/scripts/generate_code_owner_mappings.py --help -Sample output:: - - # Copy results into appropriate config yml file. - CODE_OWNER_MAPPINGS: - team-blue: - - badges - team-red: - - xblock_django - - openedx.core.djangoapps.xblock """ -import os -import re import csv import click +import os +import re + +# Maps edx-platform installed Django apps to the edx repo that contains +# the app code. +EDX_REPO_APPS = { + 'bulk_grades': 'https://github.com/edx/edx-bulk-grades', + 'coaching': 'https://github.com/edx/platform-plugin-coaching', + 'completion': 'https://github.com/edx/completion', + 'config_models': 'https://github.com/edx/django-config-models', + 'consent': 'https://github.com/edx/edx-enterprise', + 'csrf': 'https://github.com/edx/edx-drf-extensions', + 'edx_proctoring': 'https://github.com/edx/edx-proctoring', + 'edxval': 'https://github.com/edx/edx-val', + 'enterprise': 'https://github.com/edx/edx-enterprise', + 'enterprise_learner_portal': 'https://github.com/edx/edx-enterprise', + 'help_tokens': 'https://github.com/edx/help-tokens', + 'integrated_channels': 'https://github.com/edx/edx-enterprise', + 'organizations': 'https://github.com/edx/edx-organizations', + 'search': 'https://github.com/edx/edx-search', + 'wiki': 'https://github.com/edx/django-wiki', +} + +# Maps edx-platform installed Django apps to the third-party repo that contains +# the app code. +THIRD_PARTY_APPS = { + 'django': 'https://github.com/django/django', + 'django_object_actions': 'https://github.com/crccheck/django-object-actions', + 'drf_yasg': 'https://github.com/axnsan12/drf-yasg', + 'lx_pathway_plugin': 'https://github.com/open-craft/lx-pathway-plugin', + 'simple_history': 'https://github.com/treyhunner/django-simple-history', + 'social_django': 'https://github.com/python-social-auth/social-app-django', +} @click.command() +@click.option( + '--repo-csv', + help="File name of .csv file with repo ownership details.", + required=True +) @click.option( '--app-csv', help="File name of .csv file with edx-platform app ownership details.", required=True ) -def main(app_csv): +@click.option( + '--dep-csv', + help="File name of .csv file with edx-platform 3rd-party dependency ownership details.", + required=True +) +def main(repo_csv, app_csv, dep_csv): """ Reads CSV of ownership data and outputs config.yml setting to system.out. - Expected CSV format: + Expected Repo CSV format: - Path,owner.squad\n - ./lms/templates/oauth2_provider,team-red\n - ./openedx/core/djangoapps/user_authn,team-blue\n + \b + repo url,owner.squad + https://github.com/edx/edx-bulk-grades,team-red + ... + + Expected App CSV format: + + \b + Path,owner.squad + ./openedx/core/djangoapps/user_authn,team-blue + ... + + Expected 3rd-party Dependency CSV format: + + \b + repo url,owner.squad + https://github.com/django/django,team-red ... Final output only includes paths which might contain views. """ - csv_data = None - with open(app_csv, 'r') as file: + # Maps owner names to a list of dotted module paths. + # For example: { 'team-red': [ 'openedx.core.djangoapps.api_admin', 'openedx.core.djangoapps.auth_exchange' ] } + owner_to_paths_map = {} + _map_repo_apps('edx-repo', repo_csv, EDX_REPO_APPS, owner_to_paths_map) + _map_repo_apps('3rd-party', dep_csv, THIRD_PARTY_APPS, owner_to_paths_map) + _map_edx_platform_apps(app_csv, owner_to_paths_map) + + print('# Do not hand edit CODE_OWNER_MAPPINGS. Generated by {}'.format(os.path.basename(__file__))) + print('CODE_OWNER_MAPPINGS:') + for owner, path_list in sorted(owner_to_paths_map.items()): + print(" {}:".format(owner)) + path_list.sort() + for path in path_list: + print(" - {}".format(path)) + + +def _map_repo_apps(csv_type, repo_csv, app_to_repo_map, owner_to_paths_map): + """ + Reads CSV of repo ownership and uses app_to_repo_map to updates owner_to_paths_map + + Arguments: + csv_type (string): Either 'edx-repo' or '3rd-party' for warning message + repo_csv (string): File name for the edx-repo or 3rd-party repo csv + app_to_repo_map (dict): Dict mapping Django apps to repo urls + owner_to_paths_map (dict): Holds results mapping owner to paths. + + """ + with open(repo_csv, 'r') as file: csv_data = file.read() reader = csv.DictReader(csv_data.splitlines()) - team_to_paths_map = {} + csv_repo_to_owner_map = {} + for row in reader: + csv_repo_to_owner_map[row.get('repo url')] = row.get('owner.squad') + + for app, repo_url in app_to_repo_map.items(): + owner = csv_repo_to_owner_map.get(repo_url, None) + if owner: + if owner not in owner_to_paths_map: + owner_to_paths_map[owner] = [] + owner_to_paths_map[owner].append(app) + else: + print('WARNING: Repo {} was not found in {} csv. Needed for app {}.'.format(repo_url, csv_type, app)) + + +def _map_edx_platform_apps(app_csv, owner_to_paths_map): + """ + Reads CSV of edx-platform app ownership and updates mappings + """ + with open(app_csv, 'r') as file: + csv_data = file.read() + reader = csv.DictReader(csv_data.splitlines()) for row in reader: path = row.get('Path') - team = row.get('owner.squad') + owner = row.get('owner.squad') # add paths that may have views may_have_views = re.match(r'.*djangoapps', path) or re.match(r'[./]*openedx\/features', path) @@ -69,20 +158,12 @@ def main(app_csv): path = path.replace('/', '.') # convert path to dotted module name # skip catch-alls to ensure everything is properly mapped - if path in ('common,djangoapps', 'lms.djangoapps', 'openedx.core.djangoapps', 'openedx.features'): + if path in ('common.djangoapps', 'lms.djangoapps', 'openedx.core.djangoapps', 'openedx.features'): continue - if team not in team_to_paths_map: - team_to_paths_map[team] = [] - team_to_paths_map[team].append(path) - - print('# Do not hand edit CODE_OWNER_MAPPINGS. Generated by {}'.format(os.path.basename(__file__))) - print('CODE_OWNER_MAPPINGS:') - for team, path_list in sorted(team_to_paths_map.items()): - print(" {}:".format(team)) - path_list.sort() - for path in path_list: - print(" - {}".format(path)) + if owner not in owner_to_paths_map: + owner_to_paths_map[owner] = [] + owner_to_paths_map[owner].append(path) if __name__ == "__main__":