ARCHBOM-1247: ownership mapping for external apps (#24148)

- add clean script to help list unmapped Django apps.
- use clean script to source new app mappings.
- update generate script to include 3rd-party app ownership.

ARCHBOM-1247

lms/djangoapps/monitoring/scripts/clean_unmapped_view_modules.py

@@ -0,0 +1,73 @@
+"""
+Provided a CSV of data from our monitoring system, this script outputs a unique and clean
+set of apps of unmapped view_func_modules.
+
+Context: This script was useful when first introducing ownership mapping and we had many
+apps from 3rd-party dependencies that were missed. At this point, we'd probably only
+expect 0-2 new unmapped apps, which could be cleaned manually very quickly without this
+script.
+
+Sample usage::
+
+    python lms/djangoapps/monitoring/scripts/clean_unmapped_view_modules.py --unmapped-csv "unmapped-apps.csv"
+
+Or for more details::
+
+    python lms/djangoapps/monitoring/scripts/clean_unmapped_view_modules.py --help
+
+
+"""
+import csv
+import click
+
+
+@click.command()
+@click.option(
+    '--unmapped-csv',
+    help="File name of .csv file with unmapped edx-platform view modules.",
+    required=True
+)
+def main(unmapped_csv):
+    """
+    Reads CSV of unmapped view_func_modules and outputs a clean list of apps to map.
+
+    NewRelic Insights Query to create CSV of unmapped modules:
+
+        \b
+        SELECT count(view_func_module) FROM Transaction
+        WHERE code_owner is null FACET view_func_module
+        SINCE 1 week ago
+        LIMIT 50
+
+        \b
+        * Increase or decrease SINCE clause as necessary based on when the mappings were last updated.
+        * Save results as CSV for use in script
+
+    Sample CSV input::
+
+        \b
+        View Func Module,View Func Modules
+        enterprise.api.v1.views,1542
+        edx_proctoring.views,116
+        social_django.views,53
+
+    Script removes duplicates in addition to providing sorted list of plain app names.
+
+    """
+    with open(unmapped_csv, 'r') as file:
+        csv_data = file.read()
+    reader = csv.DictReader(csv_data.splitlines())
+
+    clean_apps_set = set()
+    for row in reader:
+        path = row.get('View Func Module')
+        path_parts = path.split('.')
+        clean_apps_set.add(path_parts[0])
+
+    print('# Move into generate_code_owner_mappings.py and complete mappings.')
+    for clean_app in sorted(clean_apps_set):
+        print(clean_app)
+
+
+if __name__ == "__main__":
+    main()  # pylint: disable=no-value-for-parameter

lms/djangoapps/monitoring/scripts/generate_code_owner_mappings.py

@@ -3,61 +3,150 @@ This script generates code owner mappings for monitoring LMS.
 
 Sample usage::
 
-    python lms/djangoapps/monitoring/scripts/generate_code_owner_mappings.py --app-csv "edx-platform Apps Ownership.csv"
+    python lms/djangoapps/monitoring/scripts/generate_code_owner_mappings.py --repo-csv "Individual Repo Ownership.csv" --app-csv "edx-platform Apps Ownership.csv" --dep-csv "edx-platform 3rd-party Ownership.csv"
 
-Sample CSV input::
+Or for more details::
 
-    Path,owner.squad
-    ./common/djangoapps/xblock_django,team-red
-    ./openedx/core/djangoapps/xblock,team-red
-    ./lms/djangoapps/badges,team-blue
+    python lms/djangoapps/monitoring/scripts/generate_code_owner_mappings.py --help
 
-Sample output::
-
-    # Copy results into appropriate config yml file.
-    CODE_OWNER_MAPPINGS:
-      team-blue:
-      - badges
-      team-red:
-      - xblock_django
-      - openedx.core.djangoapps.xblock
 
 """
-import os
-import re
 import csv
 import click
+import os
+import re
+
+# Maps edx-platform installed Django apps to the edx repo that contains
+# the app code.
+EDX_REPO_APPS = {
+    'bulk_grades': 'https://github.com/edx/edx-bulk-grades',
+    'coaching': 'https://github.com/edx/platform-plugin-coaching',
+    'completion': 'https://github.com/edx/completion',
+    'config_models': 'https://github.com/edx/django-config-models',
+    'consent': 'https://github.com/edx/edx-enterprise',
+    'csrf': 'https://github.com/edx/edx-drf-extensions',
+    'edx_proctoring': 'https://github.com/edx/edx-proctoring',
+    'edxval': 'https://github.com/edx/edx-val',
+    'enterprise': 'https://github.com/edx/edx-enterprise',
+    'enterprise_learner_portal': 'https://github.com/edx/edx-enterprise',
+    'help_tokens': 'https://github.com/edx/help-tokens',
+    'integrated_channels': 'https://github.com/edx/edx-enterprise',
+    'organizations': 'https://github.com/edx/edx-organizations',
+    'search': 'https://github.com/edx/edx-search',
+    'wiki': 'https://github.com/edx/django-wiki',
+}
+
+# Maps edx-platform installed Django apps to the third-party repo that contains
+# the app code.
+THIRD_PARTY_APPS = {
+    'django': 'https://github.com/django/django',
+    'django_object_actions': 'https://github.com/crccheck/django-object-actions',
+    'drf_yasg': 'https://github.com/axnsan12/drf-yasg',
+    'lx_pathway_plugin': 'https://github.com/open-craft/lx-pathway-plugin',
+    'simple_history': 'https://github.com/treyhunner/django-simple-history',
+    'social_django': 'https://github.com/python-social-auth/social-app-django',
+}
 
 
 @click.command()
+@click.option(
+    '--repo-csv',
+    help="File name of .csv file with repo ownership details.",
+    required=True
+)
 @click.option(
     '--app-csv',
     help="File name of .csv file with edx-platform app ownership details.",
     required=True
 )
-def main(app_csv):
+@click.option(
+    '--dep-csv',
+    help="File name of .csv file with edx-platform 3rd-party dependency ownership details.",
+    required=True
+)
+def main(repo_csv, app_csv, dep_csv):
     """
     Reads CSV of ownership data and outputs config.yml setting to system.out.
 
-    Expected CSV format:
+    Expected Repo CSV format:
 
-        Path,owner.squad\n
-        ./lms/templates/oauth2_provider,team-red\n
-        ./openedx/core/djangoapps/user_authn,team-blue\n
+        \b
+        repo url,owner.squad
+        https://github.com/edx/edx-bulk-grades,team-red
+        ...
+
+    Expected App CSV format:
+
+        \b
+        Path,owner.squad
+        ./openedx/core/djangoapps/user_authn,team-blue
+        ...
+
+    Expected 3rd-party Dependency CSV format:
+
+        \b
+        repo url,owner.squad
+        https://github.com/django/django,team-red
         ...
 
     Final output only includes paths which might contain views.
 
     """
-    csv_data = None
-    with open(app_csv, 'r') as file:
+    # Maps owner names to a list of dotted module paths.
+    # For example: { 'team-red': [ 'openedx.core.djangoapps.api_admin', 'openedx.core.djangoapps.auth_exchange' ] }
+    owner_to_paths_map = {}
+    _map_repo_apps('edx-repo', repo_csv, EDX_REPO_APPS, owner_to_paths_map)
+    _map_repo_apps('3rd-party', dep_csv, THIRD_PARTY_APPS, owner_to_paths_map)
+    _map_edx_platform_apps(app_csv, owner_to_paths_map)
+
+    print('# Do not hand edit CODE_OWNER_MAPPINGS. Generated by {}'.format(os.path.basename(__file__)))
+    print('CODE_OWNER_MAPPINGS:')
+    for owner, path_list in sorted(owner_to_paths_map.items()):
+        print("  {}:".format(owner))
+        path_list.sort()
+        for path in path_list:
+            print("  - {}".format(path))
+
+
+def _map_repo_apps(csv_type, repo_csv, app_to_repo_map, owner_to_paths_map):
+    """
+    Reads CSV of repo ownership and uses app_to_repo_map to updates owner_to_paths_map
+
+    Arguments:
+        csv_type (string): Either 'edx-repo' or '3rd-party' for warning message
+        repo_csv (string): File name for the edx-repo or 3rd-party repo csv
+        app_to_repo_map (dict): Dict mapping Django apps to repo urls
+        owner_to_paths_map (dict): Holds results mapping owner to paths.
+
+    """
+    with open(repo_csv, 'r') as file:
         csv_data = file.read()
     reader = csv.DictReader(csv_data.splitlines())
 
-    team_to_paths_map = {}
+    csv_repo_to_owner_map = {}
+    for row in reader:
+        csv_repo_to_owner_map[row.get('repo url')] = row.get('owner.squad')
+
+    for app, repo_url in app_to_repo_map.items():
+        owner = csv_repo_to_owner_map.get(repo_url, None)
+        if owner:
+            if owner not in owner_to_paths_map:
+                owner_to_paths_map[owner] = []
+            owner_to_paths_map[owner].append(app)
+        else:
+            print('WARNING: Repo {} was not found in {} csv. Needed for app {}.'.format(repo_url, csv_type, app))
+
+
+def _map_edx_platform_apps(app_csv, owner_to_paths_map):
+    """
+    Reads CSV of edx-platform app ownership and updates mappings
+    """
+    with open(app_csv, 'r') as file:
+        csv_data = file.read()
+    reader = csv.DictReader(csv_data.splitlines())
     for row in reader:
         path = row.get('Path')
-        team = row.get('owner.squad')
+        owner = row.get('owner.squad')
 
         # add paths that may have views
         may_have_views = re.match(r'.*djangoapps', path) or re.match(r'[./]*openedx\/features', path)
@@ -69,20 +158,12 @@ def main(app_csv):
             path = path.replace('/', '.')  # convert path to dotted module name
 
             # skip catch-alls to ensure everything is properly mapped
-            if path in ('common,djangoapps', 'lms.djangoapps', 'openedx.core.djangoapps', 'openedx.features'):
+            if path in ('common.djangoapps', 'lms.djangoapps', 'openedx.core.djangoapps', 'openedx.features'):
                 continue
 
-            if team not in team_to_paths_map:
-                team_to_paths_map[team] = []
-            team_to_paths_map[team].append(path)
-
-    print('# Do not hand edit CODE_OWNER_MAPPINGS. Generated by {}'.format(os.path.basename(__file__)))
-    print('CODE_OWNER_MAPPINGS:')
-    for team, path_list in sorted(team_to_paths_map.items()):
-        print("  {}:".format(team))
-        path_list.sort()
-        for path in path_list:
-            print("  - {}".format(path))
+            if owner not in owner_to_paths_map:
+                owner_to_paths_map[owner] = []
+            owner_to_paths_map[owner].append(path)
 
 
 if __name__ == "__main__":