diff --git a/lms/djangoapps/monitoring/scripts/generate_code_owner_mappings.py b/lms/djangoapps/monitoring/scripts/generate_code_owner_mappings.py index 0913e553df..cd802df905 100644 --- a/lms/djangoapps/monitoring/scripts/generate_code_owner_mappings.py +++ b/lms/djangoapps/monitoring/scripts/generate_code_owner_mappings.py @@ -3,7 +3,7 @@ This script generates code owner mappings for monitoring LMS. Sample usage:: - python lms/djangoapps/monitoring/scripts/generate_code_owner_mappings.py --repo-csv "Individual Repo Ownership.csv" --app-csv "edx-platform Apps Ownership.csv" --dep-csv "edx-platform 3rd-party Ownership.csv" + python lms/djangoapps/monitoring/scripts/generate_code_owner_mappings.py --repo-csv "Own Repos.csv" --app-csv "Own edx-platform Apps.csv" --dep-csv "Own edx-platform Libs.csv" Or for more details:: @@ -92,12 +92,15 @@ def main(repo_csv, app_csv, dep_csv): Final output only includes paths which might contain views. """ + # Maps theme name to a list of code owners in the theme, and squad to full code owner name. + # Code owner is a string combining theme and squad information. + owner_map = {'theme_to_owners_map': {}, 'squad_to_theme_map': {}} # Maps owner names to a list of dotted module paths. # For example: { 'team-red': [ 'openedx.core.djangoapps.api_admin', 'openedx.core.djangoapps.auth_exchange' ] } owner_to_paths_map = {} - _map_repo_apps('edx-repo', repo_csv, EDX_REPO_APPS, owner_to_paths_map) - _map_repo_apps('3rd-party', dep_csv, THIRD_PARTY_APPS, owner_to_paths_map) - _map_edx_platform_apps(app_csv, owner_to_paths_map) + _map_repo_apps('edx-repo', repo_csv, EDX_REPO_APPS, owner_map, owner_to_paths_map) + _map_repo_apps('3rd-party', dep_csv, THIRD_PARTY_APPS, owner_map, owner_to_paths_map) + _map_edx_platform_apps(app_csv, owner_map, owner_to_paths_map) print('# Do not hand edit CODE_OWNER_MAPPINGS. Generated by {}'.format(os.path.basename(__file__))) print('CODE_OWNER_MAPPINGS:') @@ -107,16 +110,30 @@ def main(repo_csv, app_csv, dep_csv): for path in path_list: print(" - {}".format(path)) + owner_with_mappings_set = set(owner_to_paths_map.keys()) + print('# Do not hand edit CODE_OWNER_THEMES. Generated by {}'.format(os.path.basename(__file__))) + print('CODE_OWNER_THEMES:') + for theme, owner_list in sorted(owner_map['theme_to_owners_map'].items()): + theme_owner_set = set(owner_list) + # only include the theme's list of owners that have mappings + theme_owner_with_mappings_list = list(theme_owner_set & owner_with_mappings_set) + if theme_owner_with_mappings_list: + print(" {}:".format(theme)) + theme_owner_with_mappings_list.sort() + for owner in theme_owner_with_mappings_list: + print(" - {}".format(owner)) -def _map_repo_apps(csv_type, repo_csv, app_to_repo_map, owner_to_paths_map): + +def _map_repo_apps(csv_type, repo_csv, app_to_repo_map, owner_map, owner_to_paths_map): """ - Reads CSV of repo ownership and uses app_to_repo_map to updates owner_to_paths_map + Reads CSV of repo ownership and uses app_to_repo_map to update owner_map and owner_to_paths_map Arguments: csv_type (string): Either 'edx-repo' or '3rd-party' for warning message repo_csv (string): File name for the edx-repo or 3rd-party repo csv app_to_repo_map (dict): Dict mapping Django apps to repo urls - owner_to_paths_map (dict): Holds results mapping owner to paths. + owner_map (dict): Dict of owner details + owner_to_paths_map (dict): Holds results mapping owner to paths """ with open(repo_csv, 'r') as file: @@ -125,7 +142,8 @@ def _map_repo_apps(csv_type, repo_csv, app_to_repo_map, owner_to_paths_map): csv_repo_to_owner_map = {} for row in reader: - csv_repo_to_owner_map[row.get('repo url')] = row.get('owner.squad') + owner = _get_and_map_code_owner(row, owner_map) + csv_repo_to_owner_map[row.get('repo url')] = owner for app, repo_url in app_to_repo_map.items(): owner = csv_repo_to_owner_map.get(repo_url, None) @@ -137,7 +155,7 @@ def _map_repo_apps(csv_type, repo_csv, app_to_repo_map, owner_to_paths_map): print('WARNING: Repo {} was not found in {} csv. Needed for app {}.'.format(repo_url, csv_type, app)) -def _map_edx_platform_apps(app_csv, owner_to_paths_map): +def _map_edx_platform_apps(app_csv, owner_map, owner_to_paths_map): """ Reads CSV of edx-platform app ownership and updates mappings """ @@ -146,7 +164,7 @@ def _map_edx_platform_apps(app_csv, owner_to_paths_map): reader = csv.DictReader(csv_data.splitlines()) for row in reader: path = row.get('Path') - owner = row.get('owner.squad') + owner = _get_and_map_code_owner(row, owner_map) # add paths that may have views may_have_views = re.match(r'.*djangoapps', path) or re.match(r'[./]*openedx\/features', path) @@ -166,5 +184,48 @@ def _map_edx_platform_apps(app_csv, owner_to_paths_map): owner_to_paths_map[owner].append(path) +def _get_and_map_code_owner(row, owner_map): + """ + From a csv row, takes the theme and squad, update ownership maps, and return the code_owner. + + Will also warn if the squad appears in multiple themes. + + Arguments: + row: A csv row that should have 'owner.theme' and 'owner.squad'. + owner_map: A dict with 'theme_to_owners_map' and 'squad_to_theme_map' keys. + + Returns: + The code_owner for the row. This is made from the theme+squad (or squad if there is no theme). + + """ + theme = row.get('owner.theme') + squad = row.get('owner.squad') + assert squad, 'Csv row is missing required owner.squad: %s' % row + + # use lower case names only + squad = squad.lower() + if theme: + theme = theme.lower() + + owner = '{}-{}'.format(theme, squad) if theme else squad + theme = theme or squad + + if squad not in owner_map['squad_to_theme_map']: + # store the theme for each squad for a later data integrity check + owner_map['squad_to_theme_map'][squad] = theme + + # add to the list of owners for each theme + if theme not in owner_map['theme_to_owners_map']: + owner_map['theme_to_owners_map'][theme] = [] + owner_map['theme_to_owners_map'][theme].append(owner) + + # assert that squads have a unique theme. otherwise we have a data integrity issues in the csv. + assert owner_map['squad_to_theme_map'][squad] == theme, \ + 'Squad %s is associated with theme %s in row %s, but theme %s elsewhere in the csv.' % \ + (squad, theme, row, owner_map['squad_to_theme_map'][squad]) + + return owner + + if __name__ == "__main__": main() # pylint: disable=no-value-for-parameter