Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "WIP:Cache course runs to programs"

Browse Source
This commit is contained in:
emma-green
2019-02-20 12:03:07 -05:00
committed by GitHub
parent 8250373ddf
commit cbf3f78325
15 changed files with 48 additions and 347 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
openedx/core/djangoapps/catalog/cache.py
View File

@@ -9,6 +9,3 @@ PATHWAY_CACHE_KEY_TPL = 'pathway-{id}'
# Cache key used to locate an item containing a list of all credit pathway ids for a site.
SITE_PATHWAY_IDS_CACHE_KEY_TPL = 'pathway-ids-{domain}'
# Template used to create cache keys for individual courses to program uuids.
COURSE_PROGRAMS_CACHE_KEY_TPL = 'course-programs-{course_run_id}'

52
openedx/core/djangoapps/catalog/management/commands/cache_programs.py
View File

@@ -4,14 +4,13 @@ import sys
from django.contrib.auth import get_user_model
from django.contrib.sites.models import Site
from django.core.cache import cache
from django.core.management import BaseCommand, CommandError
from django.core.management import BaseCommand
from openedx.core.djangoapps.catalog.cache import (
COURSE_PROGRAMS_CACHE_KEY_TPL,
PATHWAY_CACHE_KEY_TPL,
PROGRAM_CACHE_KEY_TPL,
SITE_PATHWAY_IDS_CACHE_KEY_TPL,
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL,
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
)
from openedx.core.djangoapps.catalog.models import CatalogIntegration
from openedx.core.djangoapps.catalog.utils import create_catalog_api_client
@@ -47,7 +46,6 @@ class Command(BaseCommand):
programs = {}
pathways = {}
courses = {}
for site in Site.objects.all():
site_config = getattr(site, 'configuration', None)
if site_config is None or not site_config.get_value('COURSE_CATALOG_API_URL'):
@@ -62,19 +60,12 @@ class Command(BaseCommand):
new_pathways, pathways_failed = self.get_pathways(client, site)
new_pathways, new_programs, pathway_processing_failed = self.process_pathways(site, new_pathways,
new_programs)
new_courses, courses_failed = self.get_courses(client, site)
failure = any([
program_uuids_failed,
program_details_failed,
pathways_failed,
pathway_processing_failed,
courses_failed,
])
if program_uuids_failed or program_details_failed or pathways_failed or pathway_processing_failed:
failure = True
programs.update(new_programs)
pathways.update(new_pathways)
courses.update(new_courses)
logger.info(u'Caching UUIDs for {total} programs for site {site_name}.'.format(
total=len(uuids),
@@ -99,15 +90,10 @@ class Command(BaseCommand):
successful_pathways=successful_pathways))
cache.set_many(pathways, None)
successful_courses = len(courses)
logger.info('Caching programs uuids for {successful_courses} courses.'.format(
successful_courses=successful_courses))
cache.set_many(courses, None)
if failure:
# This will fail a Jenkins job running this command, letting site
# operators know that there was a problem.
raise CommandError("Caching program information failed")
sys.exit(1)
def get_site_program_uuids(self, client, site):
failure = False
@@ -202,31 +188,3 @@ class Command(BaseCommand):
logger.exception(u'Failed to process pathways for {domain}'.format(domain=site.domain))
failure = True
return processed_pathways, programs, failure
def get_courses(self, client, site):
"""
Get all courses for the current client
"""
failure = False
courses = {}
try:
logger.info('Requesting courses for {domain}.'.format(domain=site.domain))
next_page = 1
while next_page:
new_courses = client.course_runs.get(exclude_utm=1, page=next_page)
courses.update({
COURSE_PROGRAMS_CACHE_KEY_TPL.format(course_run_id=cr['key']):
[pu['uuid'] for pu in cr['programs']]
for cr in new_courses['results']
})
next_page = next_page + 1 if new_courses['next'] else None
except: # pylint: disable=bare-except
logger.exception('Failed to retrieve courses for site: {domain}.'.format(domain=site.domain))
failure = True
logger.info('Received {total} courses for site {domain}'.format(
total=len(courses),
domain=site.domain
))
return courses, failure

165
openedx/core/djangoapps/catalog/management/commands/tests/test_cache_programs.py
View File

@@ -2,10 +2,9 @@ import json
import httpretty
from django.core.cache import cache
from django.core.management import CommandError, call_command
from django.core.management import call_command
from openedx.core.djangoapps.catalog.cache import (
COURSE_PROGRAMS_CACHE_KEY_TPL,
PATHWAY_CACHE_KEY_TPL,
PROGRAM_CACHE_KEY_TPL,
SITE_PATHWAY_IDS_CACHE_KEY_TPL,
@@ -17,12 +16,10 @@ from openedx.core.djangoapps.site_configuration.tests.mixins import SiteMixin
from openedx.core.djangolib.testing.utils import CacheIsolationTestCase, skip_unless_lms
from student.tests.factories import UserFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
@skip_unless_lms
@httpretty.activate
class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, ModuleStoreTestCase, SiteMixin):
class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMixin):
ENABLED_CACHES = ['default']
def setUp(self):
@@ -42,25 +39,10 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, ModuleS
self.list_url = self.catalog_integration.get_internal_api_url().rstrip('/') + '/programs/'
self.detail_tpl = self.list_url.rstrip('/') + '/{uuid}/'
self.pathway_url = self.catalog_integration.get_internal_api_url().rstrip('/') + '/pathways/'
self.course_run_url = self.catalog_integration.get_internal_api_url().rstrip('/') + '/course_runs/'
self.programs = ProgramFactory.create_batch(3)
self.pathways = PathwayFactory.create_batch(3)
# Build the course run list to look like the course run api. We want to start from the program
# list so that the courses are associated with the programs.
course_run_dict = {}
for program in self.programs:
program_uuid = program['uuid']
for course in program['courses']:
for course_run in course['course_runs']:
course_run_key = course_run['key']
if course_run_key in course_run_dict:
course_run_dict[course_run_key]['programs'] += {'uuid': program_uuid}
else:
course_run_dict[course_run_key] = {'key': course_run_key, 'programs': [{'uuid': program_uuid}]}
self.course_runs = course_run_dict.values()
for pathway in self.pathways:
self.programs += pathway['programs']
@@ -139,36 +121,6 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, ModuleS
content_type='application/json',
)
def mock_courses(self, course_runs, page_number=1, final=True):
"""
Mock the data for discovery's course_run endpoint
"""
def course_run_callback(request, uri, headers): # pylint: disable=unused-argument
"""
Mocks response
"""
expected = {
'exclude_utm': ['1'],
'page': [str(page_number)],
}
self.assertEqual(request.querystring, expected)
body = {
'count': len(course_runs),
'next': None if final else 'more', # we don't actually parse this value
'prev': None,
'results': course_runs
}
return (200, headers, json.dumps(body))
httpretty.register_uri(
httpretty.GET,
self.course_run_url,
body=course_run_callback,
content_type='application/json',
)
def test_handle_programs(self):
"""
Verify that the command requests and caches program UUIDs and details.
@@ -185,7 +137,6 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, ModuleS
self.mock_list()
self.mock_pathways(self.pathways)
self.mock_courses(self.course_runs)
for uuid in self.uuids:
program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
@@ -233,7 +184,6 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, ModuleS
self.mock_list()
self.mock_pathways(self.pathways)
self.mock_courses(self.course_runs)
for uuid in self.uuids:
program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
@@ -280,7 +230,6 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, ModuleS
}
self.mock_list()
self.mock_courses(self.course_runs)
for uuid in self.uuids:
program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
self.mock_detail(uuid, program)
@@ -320,89 +269,6 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, ModuleS
self.assertEqual(pathway, pathways_dict[key])
def test_handle_courses(self):
"""
Verify that the command requests and caches course to program uuids
"""
UserFactory(username=self.catalog_integration.service_username)
programs = {
PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']): program for program in self.programs
}
courses = {
COURSE_PROGRAMS_CACHE_KEY_TPL.format(course_run_id=course['key']):
[pu['uuid'] for pu in course['programs']]
for course in self.course_runs
}
self.mock_list()
self.mock_pathways(self.pathways)
self.mock_courses(self.course_runs)
for uuid in self.uuids:
program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
self.mock_detail(uuid, program)
call_command('cache_programs')
cached_courses = cache.get_many(list(courses.keys()))
self.assertEqual(
set(cached_courses),
set(courses)
)
# We can't use a set comparison here because these values are dictionaries
# and aren't hashable. We've already verified that all pathways came out
# of the cache above, so all we need to do here is verify the accuracy of
# the data itself.
for key, course in cached_courses.items():
self.assertEqual(course, courses[key])
def test_handle_courses_multiple_pages(self):
"""
Verify that the command requests and caches course to program uuids
"""
UserFactory(username=self.catalog_integration.service_username)
programs = {
PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']): program for program in self.programs
}
courses = {
COURSE_PROGRAMS_CACHE_KEY_TPL.format(course_run_id=course['key']):
[pu['uuid'] for pu in course['programs']]
for course in self.course_runs
}
self.mock_list()
self.mock_pathways(self.pathways)
# mock 3 pages of course_runs, starting at the last
self.mock_courses(self.course_runs[20:], page_number=3, final=True)
self.mock_courses(self.course_runs[10:20], page_number=2, final=False)
self.mock_courses(self.course_runs[:10], page_number=1, final=False)
for uuid in self.uuids:
program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
self.mock_detail(uuid, program)
call_command('cache_programs')
cached_courses = cache.get_many(list(courses.keys()))
self.assertEqual(
set(cached_courses),
set(courses)
)
# We can't use a set comparison here because these values are dictionaries
# and aren't hashable. We've already verified that all pathways came out
# of the cache above, so all we need to do here is verify the accuracy of
# the data itself.
for key, course in cached_courses.items():
self.assertEqual(course, courses[key])
def test_handle_missing_service_user(self):
"""
Verify that the command raises an exception when run without a service
@@ -421,9 +287,9 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, ModuleS
"""
UserFactory(username=self.catalog_integration.service_username)
with self.assertRaises(CommandError) as context:
with self.assertRaises(SystemExit) as context:
call_command('cache_programs')
self.assertEqual(str(context.exception), "Caching program information failed")
self.assertEqual(context.exception.code, 1)
cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
self.assertEqual(cached_uuids, [])
@@ -444,9 +310,9 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, ModuleS
program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
self.mock_detail(uuid, program)
with self.assertRaises(CommandError) as context:
with self.assertRaises(SystemExit) as context:
call_command('cache_programs')
self.assertEqual(str(context.exception), "Caching program information failed")
self.assertEqual(context.exception.code, 1)
cached_pathways = cache.get(SITE_PATHWAY_IDS_CACHE_KEY_TPL.format(domain=self.site_domain))
self.assertEqual(cached_pathways, [])
@@ -472,10 +338,10 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, ModuleS
program = partial_programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
self.mock_detail(uuid, program)
with self.assertRaises(CommandError) as context:
with self.assertRaises(SystemExit) as context:
call_command('cache_programs')
self.assertEqual(str(context.exception), "Caching program information failed")
self.assertEqual(context.exception.code, 1)
cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
self.assertEqual(
@@ -495,18 +361,3 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, ModuleS
# cached programs have a pathways field added to them, remove before comparing
del program['pathway_ids']
self.assertEqual(program, partial_programs[key])
def test_handle_missing_course_runs(self):
"""
Verify that the command raises an exception when it fails to retrieve course runs.
"""
UserFactory(username=self.catalog_integration.service_username)
self.mock_list()
# Only mock out the first page of courses. Pages 2 and 3 will be missing.
self.mock_courses(self.course_runs[:10], page_number=1, final=False)
with self.assertRaises(CommandError) as context:
call_command('cache_programs')
self.assertEqual(str(context.exception), "Caching program information failed")

20
openedx/core/djangoapps/catalog/tests/factories.py
View File

@@ -211,26 +211,6 @@ class ProgramFactory(DictFactoryBase):
weeks_to_complete = fake.random_int(1, 45)
class ProgramDescriptionFactory(DictFactoryBase):
uuid = factory.Faker('uuid4')
title = factory.Faker('catch_phrase')
type = factory.Faker('word')
marketing_slug = factory.Faker('slug')
marketing_url = factory.Faker('url')
number_of_courses = fake.random_int(1, 10) # pylint: disable=no-member
@classmethod
def from_program(cls, program):
return cls.create(
uuid=program['uuid'],
title=program['title'],
type=program['type'],
marketing_slug=program['marketing_slug'],
marketing_url=program['marketing_url'],
number_of_courses=len(program['courses']),
)
class ProgramTypeFactory(DictFactoryBase):
name = factory.Faker('word')
logo_image = factory.LazyFunction(generate_sized_stdimage)

4
openedx/core/djangoapps/catalog/tests/test_utils.py
View File

@@ -167,7 +167,7 @@ class TestGetPrograms(CacheIsolationTestCase):
expected_program = ProgramFactory()
expected_uuid = expected_program['uuid']
self.assertEqual(get_programs(uuid=expected_uuid), None)
self.assertEqual(get_programs(self.site, uuid=expected_uuid), None)
mock_warning.assert_called_once_with(
u'Failed to get details for program {uuid} from the cache.'.format(uuid=expected_uuid)
)
@@ -179,7 +179,7 @@ class TestGetPrograms(CacheIsolationTestCase):
None
)
actual_program = get_programs(uuid=expected_uuid)
actual_program = get_programs(self.site, uuid=expected_uuid)
self.assertEqual(actual_program, expected_program)
self.assertFalse(mock_warning.called)

31
openedx/core/djangoapps/catalog/utils.py
View File

@@ -13,13 +13,9 @@ from pytz import UTC
from entitlements.utils import is_course_run_entitlement_fulfillable
from openedx.core.constants import COURSE_PUBLISHED
from openedx.core.djangoapps.catalog.cache import (
COURSE_PROGRAMS_CACHE_KEY_TPL,
PATHWAY_CACHE_KEY_TPL,
PROGRAM_CACHE_KEY_TPL,
SITE_PATHWAY_IDS_CACHE_KEY_TPL,
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
)
from openedx.core.djangoapps.catalog.cache import (PATHWAY_CACHE_KEY_TPL, PROGRAM_CACHE_KEY_TPL,
SITE_PATHWAY_IDS_CACHE_KEY_TPL,
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL)
from openedx.core.djangoapps.catalog.models import CatalogIntegration
from openedx.core.djangoapps.oauth_dispatch.jwt import create_jwt_for_user
from openedx.core.lib.edx_api_utils import get_edx_api_data
@@ -79,23 +75,21 @@ def check_catalog_integration_and_get_user(error_message_field):
return None, catalog_integration
def get_programs(site=None, uuid=None, course=None): # pylint: disable=redefined-outer-name
def get_programs(site, uuid=None):
"""Read programs from the cache.
The cache is populated by a management command, cache_programs.
Keyword Arguments:
Arguments:
site (Site): django.contrib.sites.models object
Keyword Arguments:
uuid (string): UUID identifying a specific program to read from the cache.
course (string): course id identifying a specific course run to read from the cache.
Returns:
list of dict, representing programs.
dict, if a specific program is requested.
"""
if len([arg for arg in (site, uuid, course) if arg is not None]) != 1:
raise TypeError('get_programs takes exactly one argument')
missing_details_msg_tpl = u'Failed to get details for program {uuid} from the cache.'
if uuid:
@@ -104,14 +98,9 @@ def get_programs(site=None, uuid=None, course=None): # pylint: disable=redefine
logger.warning(missing_details_msg_tpl.format(uuid=uuid))
return program
elif course:
uuids = cache.get(COURSE_PROGRAMS_CACHE_KEY_TPL.format(course_run_id=course))
if not uuids:
logger.warning(missing_details_msg_tpl.format(course=course))
else:
uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), [])
if not uuids:
logger.warning(u'Failed to get program UUIDs from the cache for site {}.'.format(site.domain))
uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), [])
if not uuids:
logger.warning(u'Failed to get program UUIDs from the cache for site {}.'.format(site.domain))
programs = cache.get_many([PROGRAM_CACHE_KEY_TPL.format(uuid=uuid) for uuid in uuids])
programs = list(programs.values())

2
openedx/core/djangoapps/programs/utils.py
View File

@@ -109,7 +109,7 @@ class ProgramProgressMeter(object):
self.course_grade_factory = CourseGradeFactory()
if uuid:
self.programs = [get_programs(uuid=uuid)]
self.programs = [get_programs(self.site, uuid=uuid)]
else:
self.programs = attach_program_detail_url(get_programs(self.site), self.mobile_only)

6
openedx/core/djangoapps/user_api/management/commands/migrate_user_profile_langs.py
View File

@@ -99,8 +99,4 @@ class Command(BaseCommand):
end += chunk_size
sleep(sleep_time_secs)
print('Finished! Updated {} total preferences from {} to {}'.format(
updated_count,
old_lang_code,
new_lang_code
))
print('Finished! Updated {} total preferences from {} to {}'.format(updated_count, old_lang_code, new_lang_code))