From f2139bbe2f554a1c03d561797f38a2b466de7299 Mon Sep 17 00:00:00 2001
From: Emma Green <emma.green.software@gmail.com>
Date: Tue, 29 Jan 2019 10:46:35 -0500
Subject: [PATCH] add courses to programs to cache

---
 .../contentstore/views/tests/test_videos.py   |  10 +-
 common/djangoapps/student/views/dashboard.py  |   2 +-
 common/djangoapps/terrain/stubs/catalog.py    |   7 +-
 common/test/acceptance/fixtures/catalog.py    |  26 ++-
 .../acceptance/tests/lms/test_programs.py     |  61 +++++--
 .../courseware/tests/test_video_mongo.py      |   4 +-
 lms/djangoapps/courseware/views/views.py      |   2 +-
 openedx/core/djangoapps/catalog/cache.py      |   3 +
 .../management/commands/cache_programs.py     |  48 ++++-
 .../commands/tests/test_cache_programs.py     | 165 +++++++++++++++++-
 .../djangoapps/catalog/tests/factories.py     |  20 +++
 .../djangoapps/catalog/tests/test_utils.py    |   4 +-
 openedx/core/djangoapps/catalog/utils.py      |  31 ++--
 openedx/core/djangoapps/programs/utils.py     |   2 +-
 .../commands/migrate_user_profile_langs.py    |   6 +-
 15 files changed, 345 insertions(+), 46 deletions(-)

diff --git a/cms/djangoapps/contentstore/views/tests/test_videos.py b/cms/djangoapps/contentstore/views/tests/test_videos.py
index 9f97147021..cb365e8af0 100644
--- a/cms/djangoapps/contentstore/views/tests/test_videos.py
+++ b/cms/djangoapps/contentstore/views/tests/test_videos.py
@@ -958,7 +958,10 @@ class VideoImageTestCase(VideoUploadTestBase, CourseTestCase):
                 'width': 16,  # 16x9
                 'height': 9
             },
-            u'Recommended image resolution is {image_file_max_width}x{image_file_max_height}. The minimum resolution is {image_file_min_width}x{image_file_min_height}.'.format(
+            (
+                u'Recommended image resolution is {image_file_max_width}x{image_file_max_height}. '
+                u'The minimum resolution is {image_file_min_width}x{image_file_min_height}.'
+            ).format(
                 image_file_max_width=settings.VIDEO_IMAGE_MAX_WIDTH,
                 image_file_max_height=settings.VIDEO_IMAGE_MAX_HEIGHT,
                 image_file_min_width=settings.VIDEO_IMAGE_MIN_WIDTH,
@@ -970,7 +973,10 @@ class VideoImageTestCase(VideoUploadTestBase, CourseTestCase):
                 'width': settings.VIDEO_IMAGE_MIN_WIDTH - 10,
                 'height': settings.VIDEO_IMAGE_MIN_HEIGHT
             },
-            u'Recommended image resolution is {image_file_max_width}x{image_file_max_height}. The minimum resolution is {image_file_min_width}x{image_file_min_height}.'.format(
+            (
+                u'Recommended image resolution is {image_file_max_width}x{image_file_max_height}. '
+                u'The minimum resolution is {image_file_min_width}x{image_file_min_height}.'
+            ).format(
                 image_file_max_width=settings.VIDEO_IMAGE_MAX_WIDTH,
                 image_file_max_height=settings.VIDEO_IMAGE_MAX_HEIGHT,
                 image_file_min_width=settings.VIDEO_IMAGE_MIN_WIDTH,
diff --git a/common/djangoapps/student/views/dashboard.py b/common/djangoapps/student/views/dashboard.py
index 9a49dec2d5..be52bf8360 100644
--- a/common/djangoapps/student/views/dashboard.py
+++ b/common/djangoapps/student/views/dashboard.py
@@ -695,7 +695,7 @@ def student_dashboard(request):
             for program in inverted_programs.values():
                 try:
                     program_uuid = program[0]['uuid']
-                    program_data = get_programs(request.site, uuid=program_uuid)
+                    program_data = get_programs(uuid=program_uuid)
                     program_data = ProgramDataExtender(program_data, request.user).extend()
                     skus = program_data.get('skus')
                     checkout_page_url = ecommerce_service.get_checkout_page_url(*skus)
diff --git a/common/djangoapps/terrain/stubs/catalog.py b/common/djangoapps/terrain/stubs/catalog.py
index f33075a774..04c6a8b24d 100644
--- a/common/djangoapps/terrain/stubs/catalog.py
+++ b/common/djangoapps/terrain/stubs/catalog.py
@@ -15,7 +15,8 @@ class StubCatalogServiceHandler(StubHttpRequestHandler):
             r'/api/v1/programs/$': self.program_list,
             r'/api/v1/programs/([0-9a-f-]+)/$': self.program_detail,
             r'/api/v1/program_types/$': self.program_types,
-            r'/api/v1/pathways/$': self.pathways
+            r'/api/v1/pathways/$': self.pathways,
+            r'/api/v1/course_runs/$': self.course_runs,
         }
 
         if self.match_pattern(pattern_handlers):
@@ -52,6 +53,10 @@ class StubCatalogServiceHandler(StubHttpRequestHandler):
         pathways = self.server.config.get('catalog.pathways', [])
         self.send_json_response(pathways)
 
+    def course_runs(self):
+        course_runs = self.server.config.get('catalog.course_runs', {'results': [], 'next': None})
+        self.send_json_response(course_runs)
+
 
 class StubCatalogService(StubHttpService):
     HANDLER_CLASS = StubCatalogServiceHandler
diff --git a/common/test/acceptance/fixtures/catalog.py b/common/test/acceptance/fixtures/catalog.py
index 4d1bdea678..ecef37b7fa 100644
--- a/common/test/acceptance/fixtures/catalog.py
+++ b/common/test/acceptance/fixtures/catalog.py
@@ -13,7 +13,8 @@ class CatalogFixture(object):
     """
     Interface to set up mock responses from the Catalog stub server.
     """
-    def install_programs(self, programs):
+    @classmethod
+    def install_programs(cls, programs):
         """
         Stub the discovery service's program list and detail API endpoints.
 
@@ -40,7 +41,8 @@ class CatalogFixture(object):
             data={key: json.dumps(uuids)},
         )
 
-    def install_pathways(self, pathways):
+    @classmethod
+    def install_pathways(cls, pathways):
         """
         Stub the discovery service's credit pathways API endpoint
 
@@ -52,7 +54,8 @@ class CatalogFixture(object):
             data={'catalog.pathways': json.dumps({'results': pathways, 'next': None})}
         )
 
-    def install_program_types(self, program_types):
+    @classmethod
+    def install_program_types(cls, program_types):
         """
         Stub the discovery service's program type list API endpoints.
 
@@ -64,10 +67,25 @@ class CatalogFixture(object):
             data={'catalog.programs_types': json.dumps(program_types)},
         )
 
+    @classmethod
+    def install_course_runs(cls, course_runs):
+        """
+        Stub the discovery service's course run list API endpoints.
+
+        Arguments:
+            course_runs (list): A list of course runs. List endpoint will be stubbed using data from this list.
+        """
+        requests.put(
+            '{}/set_config'.format(CATALOG_STUB_URL),
+            data={'catalog.course_runs': json.dumps({'results': course_runs, 'next': None})}
+        )
+
 
 class CatalogIntegrationMixin(object):
     """Mixin providing a method used to configure the catalog integration."""
-    def set_catalog_integration(self, is_enabled=False, service_username=None):
+
+    @classmethod
+    def set_catalog_integration(cls, is_enabled=False, service_username=None):
         """Use this to change the catalog integration config model during tests."""
         ConfigModelFixture('/config/catalog', {
             'enabled': is_enabled,
diff --git a/common/test/acceptance/tests/lms/test_programs.py b/common/test/acceptance/tests/lms/test_programs.py
index 06de2b8a2f..12fa19a06b 100644
--- a/common/test/acceptance/tests/lms/test_programs.py
+++ b/common/test/acceptance/tests/lms/test_programs.py
@@ -11,7 +11,8 @@ from openedx.core.djangoapps.catalog.tests.factories import (
     CourseRunFactory,
     PathwayFactory,
     ProgramFactory,
-    ProgramTypeFactory
+    ProgramTypeFactory,
+    ProgramDescriptionFactory,
 )
 
 
@@ -24,6 +25,12 @@ class ProgramPageBase(ProgramsConfigMixin, CatalogIntegrationMixin, UniqueCourse
 
         self.programs = ProgramFactory.create_batch(3)
         self.pathways = PathwayFactory.create_batch(3)
+        self.course_runs = [
+            CourseRunFactory.create(programs=[ProgramDescriptionFactory.from_program(program)], **course_run)
+            for program in self.programs
+            for course in program['courses']
+            for course_run in course['course_runs']
+        ]
         for pathway in self.pathways:
             self.programs += pathway['programs']
 
@@ -51,18 +58,28 @@ class ProgramPageBase(ProgramsConfigMixin, CatalogIntegrationMixin, UniqueCourse
         program_type = ProgramTypeFactory()
         return ProgramFactory(courses=[course], type=program_type['name'])
 
-    def stub_catalog_api(self, programs, pathways):
+    def stub_catalog_api(self, programs=None, pathways=None, course_runs=None):
         """
         Stub the discovery service's program list and detail API endpoints, as well as
         the credit pathway list endpoint.
         """
         self.set_catalog_integration(is_enabled=True, service_username=self.username)
-        CatalogFixture().install_programs(programs)
+
+        if programs is None:
+            programs = self.programs
+
+        CatalogFixture.install_programs(programs)
 
         program_types = [program['type'] for program in programs]
-        CatalogFixture().install_program_types(program_types)
+        CatalogFixture.install_program_types(program_types)
 
-        CatalogFixture().install_pathways(pathways)
+        if pathways is None:
+            pathways = self.pathways
+        CatalogFixture.install_pathways(pathways)
+
+        if course_runs is None:
+            course_runs = self.course_runs
+        CatalogFixture.install_course_runs(course_runs)
 
     def cache_programs(self):
         """
@@ -84,7 +101,7 @@ class ProgramListingPageTest(ProgramPageBase):
     def test_no_enrollments(self):
         """Verify that no cards appear when the user has no enrollments."""
         self.auth(enroll=False)
-        self.stub_catalog_api(self.programs, self.pathways)
+        self.stub_catalog_api()
         self.cache_programs()
 
         self.listing_page.visit()
@@ -98,7 +115,7 @@ class ProgramListingPageTest(ProgramPageBase):
         but none are included in an active program.
         """
         self.auth()
-        self.stub_catalog_api(self.programs, self.pathways)
+        self.stub_catalog_api()
         self.cache_programs()
 
         self.listing_page.visit()
@@ -126,7 +143,15 @@ class ProgramListingPageA11yTest(ProgramPageBase):
             ]
         })
         self.auth(enroll=False)
-        self.stub_catalog_api(programs=[self.program], pathways=[])
+        self.stub_catalog_api(
+            programs=[self.program],
+            pathways=[],
+            course_runs=[
+                CourseRunFactory.create(programs=[ProgramDescriptionFactory.from_program(self.program)], **course_run)
+                for course in self.program['courses']
+                for course_run in course['course_runs']
+            ]
+        )
         self.cache_programs()
 
         self.listing_page.visit()
@@ -143,7 +168,15 @@ class ProgramListingPageA11yTest(ProgramPageBase):
             ]
         })
         self.auth()
-        self.stub_catalog_api(programs=[self.program], pathways=[])
+        self.stub_catalog_api(
+            programs=[self.program],
+            pathways=[],
+            course_runs=[
+                CourseRunFactory.create(programs=[ProgramDescriptionFactory.from_program(self.program)], **course_run)
+                for course in self.program['courses']
+                for course_run in course['course_runs']
+            ]
+        )
         self.cache_programs()
 
         self.listing_page.visit()
@@ -173,7 +206,15 @@ class ProgramDetailsPageA11yTest(ProgramPageBase):
             ]
         })
         self.auth()
-        self.stub_catalog_api(programs=[self.program], pathways=[])
+        self.stub_catalog_api(
+            programs=[self.program],
+            pathways=[],
+            course_runs=[
+                CourseRunFactory.create(programs=[ProgramDescriptionFactory.from_program(self.program)], **course_run)
+                for course in self.program['courses']
+                for course_run in course['course_runs']
+            ]
+        )
         self.cache_programs()
 
         self.details_page.visit()
diff --git a/lms/djangoapps/courseware/tests/test_video_mongo.py b/lms/djangoapps/courseware/tests/test_video_mongo.py
index 728d1e7da5..c17995ddb9 100644
--- a/lms/djangoapps/courseware/tests/test_video_mongo.py
+++ b/lms/djangoapps/courseware/tests/test_video_mongo.py
@@ -814,7 +814,7 @@ class TestGetHtmlMethod(BaseTestXmodule):
 
         mocked_get_video.side_effect = side_effect
 
-        SOURCE_XML = """
+        source_xml = """
             <video show_captions="true"
             display_name="A Name"
             sub="a_sub_file.srt.sjson" source="{source}"
@@ -875,7 +875,7 @@ class TestGetHtmlMethod(BaseTestXmodule):
         initial_context['metadata']['duration'] = None
 
         for data in cases:
-            DATA = SOURCE_XML.format(
+            DATA = source_xml.format(
                 download_video=data['download_video'],
                 source=data['source'],
                 sources=data['sources'],
diff --git a/lms/djangoapps/courseware/views/views.py b/lms/djangoapps/courseware/views/views.py
index 8b660bc061..e5c9d97343 100644
--- a/lms/djangoapps/courseware/views/views.py
+++ b/lms/djangoapps/courseware/views/views.py
@@ -900,7 +900,7 @@ def program_marketing(request, program_uuid):
     """
     Display the program marketing page.
     """
-    program_data = get_programs(request.site, uuid=program_uuid)
+    program_data = get_programs(uuid=program_uuid)
 
     if not program_data:
         raise Http404
diff --git a/openedx/core/djangoapps/catalog/cache.py b/openedx/core/djangoapps/catalog/cache.py
index c0111b1363..fdb77122c6 100644
--- a/openedx/core/djangoapps/catalog/cache.py
+++ b/openedx/core/djangoapps/catalog/cache.py
@@ -9,3 +9,6 @@ PATHWAY_CACHE_KEY_TPL = 'pathway-{id}'
 
 # Cache key used to locate an item containing a list of all credit pathway ids for a site.
 SITE_PATHWAY_IDS_CACHE_KEY_TPL = 'pathway-ids-{domain}'
+
+# Template used to create cache keys for individual courses to program uuids.
+COURSE_PROGRAMS_CACHE_KEY_TPL = 'course-programs-{course_run_id}'
diff --git a/openedx/core/djangoapps/catalog/management/commands/cache_programs.py b/openedx/core/djangoapps/catalog/management/commands/cache_programs.py
index 1901b0bd67..4e7208dd4c 100644
--- a/openedx/core/djangoapps/catalog/management/commands/cache_programs.py
+++ b/openedx/core/djangoapps/catalog/management/commands/cache_programs.py
@@ -7,10 +7,11 @@ from django.core.cache import cache
 from django.core.management import BaseCommand, CommandError
 
 from openedx.core.djangoapps.catalog.cache import (
+    COURSE_PROGRAMS_CACHE_KEY_TPL,
     PATHWAY_CACHE_KEY_TPL,
     PROGRAM_CACHE_KEY_TPL,
     SITE_PATHWAY_IDS_CACHE_KEY_TPL,
-    SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
+    SITE_PROGRAM_UUIDS_CACHE_KEY_TPL,
 )
 from openedx.core.djangoapps.catalog.models import CatalogIntegration
 from openedx.core.djangoapps.catalog.utils import create_catalog_api_client
@@ -46,6 +47,7 @@ class Command(BaseCommand):
 
         programs = {}
         pathways = {}
+        courses = {}
         for site in Site.objects.all():
             site_config = getattr(site, 'configuration', None)
             if site_config is None or not site_config.get_value('COURSE_CATALOG_API_URL'):
@@ -60,12 +62,19 @@ class Command(BaseCommand):
             new_pathways, pathways_failed = self.get_pathways(client, site)
             new_pathways, new_programs, pathway_processing_failed = self.process_pathways(site, new_pathways,
                                                                                           new_programs)
+            new_courses, courses_failed = self.get_courses(client, site)
 
-            if program_uuids_failed or program_details_failed or pathways_failed or pathway_processing_failed:
-                failure = True
+            failure = any([
+                program_uuids_failed,
+                program_details_failed,
+                pathways_failed,
+                pathway_processing_failed,
+                courses_failed,
+            ])
 
             programs.update(new_programs)
             pathways.update(new_pathways)
+            courses.update(new_courses)
 
             logger.info(u'Caching UUIDs for {total} programs for site {site_name}.'.format(
                 total=len(uuids),
@@ -90,6 +99,11 @@ class Command(BaseCommand):
             successful_pathways=successful_pathways))
         cache.set_many(pathways, None)
 
+        successful_courses = len(courses)
+        logger.info('Caching programs uuids for {successful_courses} courses.'.format(
+            successful_courses=successful_courses))
+        cache.set_many(courses, None)
+
         if failure:
             # This will fail a Jenkins job running this command, letting site
             # operators know that there was a problem.
@@ -188,3 +202,31 @@ class Command(BaseCommand):
                 logger.exception(u'Failed to process pathways for {domain}'.format(domain=site.domain))
                 failure = True
         return processed_pathways, programs, failure
+
+    def get_courses(self, client, site):
+        """
+        Get all courses for the current client
+        """
+        failure = False
+        courses = {}
+        try:
+            logger.info('Requesting courses for {domain}.'.format(domain=site.domain))
+            next_page = 1
+            while next_page:
+                new_courses = client.course_runs.get(exclude_utm=1, page=next_page)
+                courses.update({
+                    COURSE_PROGRAMS_CACHE_KEY_TPL.format(course_run_id=cr['key']):
+                    [pu['uuid'] for pu in cr['programs']]
+                    for cr in new_courses['results']
+                })
+
+                next_page = next_page + 1 if new_courses['next'] else None
+        except:  # pylint: disable=bare-except
+            logger.exception('Failed to retrieve courses for site: {domain}.'.format(domain=site.domain))
+            failure = True
+
+        logger.info('Received {total} courses for site {domain}'.format(
+            total=len(courses),
+            domain=site.domain
+        ))
+        return courses, failure
diff --git a/openedx/core/djangoapps/catalog/management/commands/tests/test_cache_programs.py b/openedx/core/djangoapps/catalog/management/commands/tests/test_cache_programs.py
index 499fd00ed3..24189526c1 100644
--- a/openedx/core/djangoapps/catalog/management/commands/tests/test_cache_programs.py
+++ b/openedx/core/djangoapps/catalog/management/commands/tests/test_cache_programs.py
@@ -2,9 +2,10 @@ import json
 
 import httpretty
 from django.core.cache import cache
-from django.core.management import call_command
+from django.core.management import CommandError, call_command
 
 from openedx.core.djangoapps.catalog.cache import (
+    COURSE_PROGRAMS_CACHE_KEY_TPL,
     PATHWAY_CACHE_KEY_TPL,
     PROGRAM_CACHE_KEY_TPL,
     SITE_PATHWAY_IDS_CACHE_KEY_TPL,
@@ -16,10 +17,12 @@ from openedx.core.djangoapps.site_configuration.tests.mixins import SiteMixin
 from openedx.core.djangolib.testing.utils import CacheIsolationTestCase, skip_unless_lms
 from student.tests.factories import UserFactory
 
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+
 
 @skip_unless_lms
 @httpretty.activate
-class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMixin):
+class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, ModuleStoreTestCase, SiteMixin):
     ENABLED_CACHES = ['default']
 
     def setUp(self):
@@ -39,10 +42,25 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
         self.list_url = self.catalog_integration.get_internal_api_url().rstrip('/') + '/programs/'
         self.detail_tpl = self.list_url.rstrip('/') + '/{uuid}/'
         self.pathway_url = self.catalog_integration.get_internal_api_url().rstrip('/') + '/pathways/'
+        self.course_run_url = self.catalog_integration.get_internal_api_url().rstrip('/') + '/course_runs/'
 
         self.programs = ProgramFactory.create_batch(3)
         self.pathways = PathwayFactory.create_batch(3)
 
+        # Build the course run list to look like the course run api. We want to start from the program
+        # list so that the courses are associated with the programs.
+        course_run_dict = {}
+        for program in self.programs:
+            program_uuid = program['uuid']
+            for course in program['courses']:
+                for course_run in course['course_runs']:
+                    course_run_key = course_run['key']
+                    if course_run_key in course_run_dict:
+                        course_run_dict[course_run_key]['programs'] += {'uuid': program_uuid}
+                    else:
+                        course_run_dict[course_run_key] = {'key': course_run_key, 'programs': [{'uuid': program_uuid}]}
+        self.course_runs = course_run_dict.values()
+
         for pathway in self.pathways:
             self.programs += pathway['programs']
 
@@ -121,6 +139,36 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
             content_type='application/json',
         )
 
+    def mock_courses(self, course_runs, page_number=1, final=True):
+        """
+        Mock the data for discovery's course_run endpoint
+        """
+        def course_run_callback(request, uri, headers):  # pylint: disable=unused-argument
+            """
+            Mocks response
+            """
+            expected = {
+                'exclude_utm': ['1'],
+                'page': [str(page_number)],
+            }
+            self.assertEqual(request.querystring, expected)
+
+            body = {
+                'count': len(course_runs),
+                'next': None if final else 'more',  # we don't actually parse this value
+                'prev': None,
+                'results': course_runs
+            }
+
+            return (200, headers, json.dumps(body))
+
+        httpretty.register_uri(
+            httpretty.GET,
+            self.course_run_url,
+            body=course_run_callback,
+            content_type='application/json',
+        )
+
     def test_handle_programs(self):
         """
         Verify that the command requests and caches program UUIDs and details.
@@ -137,6 +185,7 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
 
         self.mock_list()
         self.mock_pathways(self.pathways)
+        self.mock_courses(self.course_runs)
 
         for uuid in self.uuids:
             program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
@@ -184,6 +233,7 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
 
         self.mock_list()
         self.mock_pathways(self.pathways)
+        self.mock_courses(self.course_runs)
 
         for uuid in self.uuids:
             program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
@@ -230,6 +280,7 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
         }
 
         self.mock_list()
+        self.mock_courses(self.course_runs)
         for uuid in self.uuids:
             program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
             self.mock_detail(uuid, program)
@@ -269,6 +320,89 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
 
             self.assertEqual(pathway, pathways_dict[key])
 
+    def test_handle_courses(self):
+        """
+        Verify that the command requests and caches course to program uuids
+        """
+
+        UserFactory(username=self.catalog_integration.service_username)
+
+        programs = {
+            PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']): program for program in self.programs
+        }
+
+        courses = {
+            COURSE_PROGRAMS_CACHE_KEY_TPL.format(course_run_id=course['key']):
+            [pu['uuid'] for pu in course['programs']]
+            for course in self.course_runs
+        }
+
+        self.mock_list()
+        self.mock_pathways(self.pathways)
+        self.mock_courses(self.course_runs)
+
+        for uuid in self.uuids:
+            program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
+            self.mock_detail(uuid, program)
+
+        call_command('cache_programs')
+
+        cached_courses = cache.get_many(list(courses.keys()))
+        self.assertEqual(
+            set(cached_courses),
+            set(courses)
+        )
+
+        # We can't use a set comparison here because these values are dictionaries
+        # and aren't hashable. We've already verified that all pathways came out
+        # of the cache above, so all we need to do here is verify the accuracy of
+        # the data itself.
+        for key, course in cached_courses.items():
+            self.assertEqual(course, courses[key])
+
+    def test_handle_courses_multiple_pages(self):
+        """
+        Verify that the command requests and caches course to program uuids
+        """
+
+        UserFactory(username=self.catalog_integration.service_username)
+
+        programs = {
+            PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']): program for program in self.programs
+        }
+
+        courses = {
+            COURSE_PROGRAMS_CACHE_KEY_TPL.format(course_run_id=course['key']):
+            [pu['uuid'] for pu in course['programs']]
+            for course in self.course_runs
+        }
+
+        self.mock_list()
+        self.mock_pathways(self.pathways)
+        # mock 3 pages of course_runs, starting at the last
+        self.mock_courses(self.course_runs[20:], page_number=3, final=True)
+        self.mock_courses(self.course_runs[10:20], page_number=2, final=False)
+        self.mock_courses(self.course_runs[:10], page_number=1, final=False)
+
+        for uuid in self.uuids:
+            program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
+            self.mock_detail(uuid, program)
+
+        call_command('cache_programs')
+
+        cached_courses = cache.get_many(list(courses.keys()))
+        self.assertEqual(
+            set(cached_courses),
+            set(courses)
+        )
+
+        # We can't use a set comparison here because these values are dictionaries
+        # and aren't hashable. We've already verified that all pathways came out
+        # of the cache above, so all we need to do here is verify the accuracy of
+        # the data itself.
+        for key, course in cached_courses.items():
+            self.assertEqual(course, courses[key])
+
     def test_handle_missing_service_user(self):
         """
         Verify that the command raises an exception when run without a service
@@ -287,9 +421,9 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
         """
         UserFactory(username=self.catalog_integration.service_username)
 
-        with self.assertRaises(SystemExit) as context:
+        with self.assertRaises(CommandError) as context:
             call_command('cache_programs')
-        self.assertEqual(context.exception.code, 1)
+        self.assertEqual(str(context.exception), "Caching program information failed")
 
         cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
         self.assertEqual(cached_uuids, [])
@@ -310,9 +444,9 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
             program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
             self.mock_detail(uuid, program)
 
-        with self.assertRaises(SystemExit) as context:
+        with self.assertRaises(CommandError) as context:
             call_command('cache_programs')
-        self.assertEqual(context.exception.code, 1)
+        self.assertEqual(str(context.exception), "Caching program information failed")
 
         cached_pathways = cache.get(SITE_PATHWAY_IDS_CACHE_KEY_TPL.format(domain=self.site_domain))
         self.assertEqual(cached_pathways, [])
@@ -338,10 +472,10 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
             program = partial_programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
             self.mock_detail(uuid, program)
 
-        with self.assertRaises(SystemExit) as context:
+        with self.assertRaises(CommandError) as context:
             call_command('cache_programs')
 
-        self.assertEqual(context.exception.code, 1)
+        self.assertEqual(str(context.exception), "Caching program information failed")
 
         cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
         self.assertEqual(
@@ -361,3 +495,18 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
             # cached programs have a pathways field added to them, remove before comparing
             del program['pathway_ids']
             self.assertEqual(program, partial_programs[key])
+
+    def test_handle_missing_course_runs(self):
+        """
+        Verify that the command raises an exception when it fails to retrieve course runs.
+        """
+        UserFactory(username=self.catalog_integration.service_username)
+
+        self.mock_list()
+        # Only mock out the first page of courses. Pages 2 and 3 will be missing.
+        self.mock_courses(self.course_runs[:10], page_number=1, final=False)
+
+        with self.assertRaises(CommandError) as context:
+            call_command('cache_programs')
+
+        self.assertEqual(str(context.exception), "Caching program information failed")
diff --git a/openedx/core/djangoapps/catalog/tests/factories.py b/openedx/core/djangoapps/catalog/tests/factories.py
index 357ee54d38..8b498425e5 100644
--- a/openedx/core/djangoapps/catalog/tests/factories.py
+++ b/openedx/core/djangoapps/catalog/tests/factories.py
@@ -211,6 +211,26 @@ class ProgramFactory(DictFactoryBase):
     weeks_to_complete = fake.random_int(1, 45)
 
 
+class ProgramDescriptionFactory(DictFactoryBase):
+    uuid = factory.Faker('uuid4')
+    title = factory.Faker('catch_phrase')
+    type = factory.Faker('word')
+    marketing_slug = factory.Faker('slug')
+    marketing_url = factory.Faker('url')
+    number_of_courses = fake.random_int(1, 10)  # pylint: disable=no-member
+
+    @classmethod
+    def from_program(cls, program):
+        return cls.create(
+            uuid=program['uuid'],
+            title=program['title'],
+            type=program['type'],
+            marketing_slug=program['marketing_slug'],
+            marketing_url=program['marketing_url'],
+            number_of_courses=len(program['courses']),
+        )
+
+
 class ProgramTypeFactory(DictFactoryBase):
     name = factory.Faker('word')
     logo_image = factory.LazyFunction(generate_sized_stdimage)
diff --git a/openedx/core/djangoapps/catalog/tests/test_utils.py b/openedx/core/djangoapps/catalog/tests/test_utils.py
index 6122cef7be..6cf498e179 100644
--- a/openedx/core/djangoapps/catalog/tests/test_utils.py
+++ b/openedx/core/djangoapps/catalog/tests/test_utils.py
@@ -167,7 +167,7 @@ class TestGetPrograms(CacheIsolationTestCase):
         expected_program = ProgramFactory()
         expected_uuid = expected_program['uuid']
 
-        self.assertEqual(get_programs(self.site, uuid=expected_uuid), None)
+        self.assertEqual(get_programs(uuid=expected_uuid), None)
         mock_warning.assert_called_once_with(
             u'Failed to get details for program {uuid} from the cache.'.format(uuid=expected_uuid)
         )
@@ -179,7 +179,7 @@ class TestGetPrograms(CacheIsolationTestCase):
             None
         )
 
-        actual_program = get_programs(self.site, uuid=expected_uuid)
+        actual_program = get_programs(uuid=expected_uuid)
         self.assertEqual(actual_program, expected_program)
         self.assertFalse(mock_warning.called)
 
diff --git a/openedx/core/djangoapps/catalog/utils.py b/openedx/core/djangoapps/catalog/utils.py
index 9fb74c0cf8..c15f44d465 100644
--- a/openedx/core/djangoapps/catalog/utils.py
+++ b/openedx/core/djangoapps/catalog/utils.py
@@ -13,9 +13,13 @@ from pytz import UTC
 
 from entitlements.utils import is_course_run_entitlement_fulfillable
 from openedx.core.constants import COURSE_PUBLISHED
-from openedx.core.djangoapps.catalog.cache import (PATHWAY_CACHE_KEY_TPL, PROGRAM_CACHE_KEY_TPL,
-                                                   SITE_PATHWAY_IDS_CACHE_KEY_TPL,
-                                                   SITE_PROGRAM_UUIDS_CACHE_KEY_TPL)
+from openedx.core.djangoapps.catalog.cache import (
+    COURSE_PROGRAMS_CACHE_KEY_TPL,
+    PATHWAY_CACHE_KEY_TPL,
+    PROGRAM_CACHE_KEY_TPL,
+    SITE_PATHWAY_IDS_CACHE_KEY_TPL,
+    SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
+)
 from openedx.core.djangoapps.catalog.models import CatalogIntegration
 from openedx.core.djangoapps.oauth_dispatch.jwt import create_jwt_for_user
 from openedx.core.lib.edx_api_utils import get_edx_api_data
@@ -75,21 +79,23 @@ def check_catalog_integration_and_get_user(error_message_field):
         return None, catalog_integration
 
 
-def get_programs(site, uuid=None):
+def get_programs(site=None, uuid=None, course=None):  # pylint: disable=redefined-outer-name
     """Read programs from the cache.
 
     The cache is populated by a management command, cache_programs.
 
-    Arguments:
-        site (Site): django.contrib.sites.models object
-
     Keyword Arguments:
+        site (Site): django.contrib.sites.models object
         uuid (string): UUID identifying a specific program to read from the cache.
+        course (string): course id identifying a specific course run to read from the cache.
 
     Returns:
         list of dict, representing programs.
         dict, if a specific program is requested.
     """
+    if len([arg for arg in (site, uuid, course) if arg is not None]) != 1:
+        raise TypeError('get_programs takes exactly one argument')
+
     missing_details_msg_tpl = u'Failed to get details for program {uuid} from the cache.'
 
     if uuid:
@@ -98,9 +104,14 @@ def get_programs(site, uuid=None):
             logger.warning(missing_details_msg_tpl.format(uuid=uuid))
 
         return program
-    uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), [])
-    if not uuids:
-        logger.warning(u'Failed to get program UUIDs from the cache for site {}.'.format(site.domain))
+    elif course:
+        uuids = cache.get(COURSE_PROGRAMS_CACHE_KEY_TPL.format(course_run_id=course))
+        if not uuids:
+            logger.warning(missing_details_msg_tpl.format(course=course))
+    else:
+        uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), [])
+        if not uuids:
+            logger.warning(u'Failed to get program UUIDs from the cache for site {}.'.format(site.domain))
 
     programs = cache.get_many([PROGRAM_CACHE_KEY_TPL.format(uuid=uuid) for uuid in uuids])
     programs = list(programs.values())
diff --git a/openedx/core/djangoapps/programs/utils.py b/openedx/core/djangoapps/programs/utils.py
index 3aa7eb1e57..8ed354dd79 100644
--- a/openedx/core/djangoapps/programs/utils.py
+++ b/openedx/core/djangoapps/programs/utils.py
@@ -109,7 +109,7 @@ class ProgramProgressMeter(object):
         self.course_grade_factory = CourseGradeFactory()
 
         if uuid:
-            self.programs = [get_programs(self.site, uuid=uuid)]
+            self.programs = [get_programs(uuid=uuid)]
         else:
             self.programs = attach_program_detail_url(get_programs(self.site), self.mobile_only)
 
diff --git a/openedx/core/djangoapps/user_api/management/commands/migrate_user_profile_langs.py b/openedx/core/djangoapps/user_api/management/commands/migrate_user_profile_langs.py
index 3183b08318..9433587475 100644
--- a/openedx/core/djangoapps/user_api/management/commands/migrate_user_profile_langs.py
+++ b/openedx/core/djangoapps/user_api/management/commands/migrate_user_profile_langs.py
@@ -99,4 +99,8 @@ class Command(BaseCommand):
             end += chunk_size
             sleep(sleep_time_secs)
 
-        print('Finished! Updated {} total preferences from {} to {}'.format(updated_count, old_lang_code, new_lang_code))
+        print('Finished! Updated {} total preferences from {} to {}'.format(
+            updated_count,
+            old_lang_code,
+            new_lang_code
+        ))