Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #18641 from edx/colelrogers/LEARNER-5922-cache-pathways

Browse Source 
Add credit pathway caching
This commit is contained in:
Cole Rogers
2018-08-01 11:13:44 -04:00
committed by GitHub
parent 600c331ade 1c05308d4a
commit 2a438df3b1
9 changed files with 435 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
common/djangoapps/terrain/stubs/catalog.py
View File

@@ -15,6 +15,7 @@ class StubCatalogServiceHandler(StubHttpRequestHandler):
r'/api/v1/programs/$': self.program_list,
r'/api/v1/programs/([0-9a-f-]+)/$': self.program_detail,
r'/api/v1/program_types/$': self.program_types,
r'/api/v1/credit_pathways/$': self.credit_pathways
}
if self.match_pattern(pattern_handlers):
@@ -47,6 +48,10 @@ class StubCatalogServiceHandler(StubHttpRequestHandler):
program_types = self.server.config.get('catalog.programs_types', [])
self.send_json_response(program_types)
def credit_pathways(self):
credit_pathways = self.server.config.get('catalog.credit_pathways', [])
self.send_json_response(credit_pathways)
class StubCatalogService(StubHttpService):
HANDLER_CLASS = StubCatalogServiceHandler

12
common/test/acceptance/fixtures/catalog.py
View File

@@ -40,6 +40,18 @@ class CatalogFixture(object):
data={key: json.dumps(uuids)},
)
def install_credit_pathways(self, credit_pathways):
"""
Stub the discovery service's credit pathways API endpoint
Arguments:
credit_pathways (list): A list of credit pathways. List endpoint will be stubbed using data from this list.
"""
requests.put(
'{}/set_config'.format(CATALOG_STUB_URL),
data={'catalog.credit_pathways': json.dumps({'results': credit_pathways})}
)
def install_program_types(self, program_types):
"""
Stub the discovery service's program type list API endpoints.

26
common/test/acceptance/tests/lms/test_programs.py
View File

@@ -11,6 +11,7 @@ from common.test.acceptance.tests.helpers import UniqueCourseTest
from openedx.core.djangoapps.catalog.tests.factories import (
CourseFactory,
CourseRunFactory,
CreditPathwayFactory,
ProgramFactory,
ProgramTypeFactory
)
@@ -24,6 +25,14 @@ class ProgramPageBase(ProgramsConfigMixin, CatalogIntegrationMixin, UniqueCourse
self.set_programs_api_configuration(is_enabled=True)
self.programs = ProgramFactory.create_batch(3)
self.credit_pathways = CreditPathwayFactory.create_batch(3)
for pathway in self.credit_pathways:
self.programs += pathway['programs']
# add some of the previously created programs to some pathways
self.credit_pathways[0]['programs'].extend([self.programs[0], self.programs[1]])
self.credit_pathways[1]['programs'].append(self.programs[0])
self.username = None
def auth(self, enroll=True):
@@ -44,9 +53,10 @@ class ProgramPageBase(ProgramsConfigMixin, CatalogIntegrationMixin, UniqueCourse
program_type = ProgramTypeFactory()
return ProgramFactory(courses=[course], type=program_type['name'])
def stub_catalog_api(self, programs):
def stub_catalog_api(self, programs, credit_pathways):
"""
Stub the discovery service's program list and detail API endpoints.
Stub the discovery service's program list and detail API endpoints, as well as
the credit pathway list endpoint.
"""
self.set_catalog_integration(is_enabled=True, service_username=self.username)
CatalogFixture().install_programs(programs)
@@ -54,6 +64,8 @@ class ProgramPageBase(ProgramsConfigMixin, CatalogIntegrationMixin, UniqueCourse
program_types = [program['type'] for program in programs]
CatalogFixture().install_program_types(program_types)
CatalogFixture().install_credit_pathways(credit_pathways)
def cache_programs(self):
"""
Populate the LMS' cache of program data.
@@ -73,7 +85,7 @@ class ProgramListingPageTest(ProgramPageBase):
def test_no_enrollments(self):
"""Verify that no cards appear when the user has no enrollments."""
self.auth(enroll=False)
self.stub_catalog_api(self.programs)
self.stub_catalog_api(self.programs, self.credit_pathways)
self.cache_programs()
self.listing_page.visit()
@@ -87,7 +99,7 @@ class ProgramListingPageTest(ProgramPageBase):
but none are included in an active program.
"""
self.auth()
self.stub_catalog_api(self.programs)
self.stub_catalog_api(self.programs, self.credit_pathways)
self.cache_programs()
self.listing_page.visit()
@@ -109,7 +121,7 @@ class ProgramListingPageA11yTest(ProgramPageBase):
def test_empty_a11y(self):
"""Test a11y of the page's empty state."""
self.auth(enroll=False)
self.stub_catalog_api(programs=[self.program])
self.stub_catalog_api(programs=[self.program], credit_pathways=[])
self.cache_programs()
self.listing_page.visit()
@@ -121,7 +133,7 @@ class ProgramListingPageA11yTest(ProgramPageBase):
def test_cards_a11y(self):
"""Test a11y when program cards are present."""
self.auth()
self.stub_catalog_api(programs=[self.program])
self.stub_catalog_api(programs=[self.program], credit_pathways=[])
self.cache_programs()
self.listing_page.visit()
@@ -145,7 +157,7 @@ class ProgramDetailsPageA11yTest(ProgramPageBase):
def test_a11y(self):
"""Test the page's a11y compliance."""
self.auth()
self.stub_catalog_api(programs=[self.program])
self.stub_catalog_api(programs=[self.program], credit_pathways=[])
self.cache_programs()
self.details_page.visit()

6
openedx/core/djangoapps/catalog/cache.py
View File

@@ -3,3 +3,9 @@ PROGRAM_CACHE_KEY_TPL = 'program-{uuid}'
# Cache key used to locate an item containing a list of all program UUIDs for a site.
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL = 'program-uuids-{domain}'
# Template used to create cache keys for individual pathways
CREDIT_PATHWAY_CACHE_KEY_TPL = 'credit-pathway-{id}'
# Cache key used to locate an item containing a list of all credit pathway ids for a site.
SITE_CREDIT_PATHWAY_IDS_CACHE_KEY_TPL = 'credit-pathway-ids-{domain}'

77
openedx/core/djangoapps/catalog/management/commands/cache_programs.py
View File

@@ -7,7 +7,9 @@ from django.core.cache import cache
from django.core.management import BaseCommand
from openedx.core.djangoapps.catalog.cache import (
CREDIT_PATHWAY_CACHE_KEY_TPL,
PROGRAM_CACHE_KEY_TPL,
SITE_CREDIT_PATHWAY_IDS_CACHE_KEY_TPL,
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
)
from openedx.core.djangoapps.catalog.models import CatalogIntegration
@@ -43,21 +45,27 @@ class Command(BaseCommand):
raise
programs = {}
pathways = {}
for site in Site.objects.all():
site_config = getattr(site, 'configuration', None)
if site_config is None or not site_config.get_value('COURSE_CATALOG_API_URL'):
logger.info('Skipping site {domain}. No configuration.'.format(domain=site.domain))
cache.set(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), [], None)
cache.set(SITE_CREDIT_PATHWAY_IDS_CACHE_KEY_TPL.format(domain=site.domain), [], None)
continue
client = create_catalog_api_client(user, site=site)
uuids, program_uuids_failed = self.get_site_program_uuids(client, site)
new_programs, program_details_failed = self.fetch_program_details(client, uuids)
new_pathways, pathways_failed = self.get_pathways(client, site)
new_pathways, new_programs, pathway_processing_failed = self.process_pathways(site, new_pathways,
new_programs)
if program_uuids_failed or program_details_failed:
if program_uuids_failed or program_details_failed or pathways_failed or pathway_processing_failed:
failure = True
programs.update(new_programs)
pathways.update(new_pathways)
logger.info('Caching UUIDs for {total} programs for site {site_name}.'.format(
total=len(uuids),
@@ -65,10 +73,23 @@ class Command(BaseCommand):
))
cache.set(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), uuids, None)
successful = len(programs)
logger.info('Caching details for {successful} programs.'.format(successful=successful))
pathway_ids = new_pathways.keys()
logger.info('Caching ids for {total} credit pathways for site {site_name}.'.format(
total=len(pathway_ids),
site_name=site.domain,
))
cache.set(SITE_CREDIT_PATHWAY_IDS_CACHE_KEY_TPL.format(domain=site.domain), pathway_ids, None)
successful_programs = len(programs)
logger.info('Caching details for {successful_programs} programs.'.format(
successful_programs=successful_programs))
cache.set_many(programs, None)
successful_pathways = len(pathways)
logger.info('Caching details for {successful_pathways} credit pathways.'.format(
successful_pathways=successful_pathways))
cache.set_many(pathways, None)
if failure:
# This will fail a Jenkins job running this command, letting site
# operators know that there was a problem.
@@ -104,9 +125,59 @@ class Command(BaseCommand):
cache_key = PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)
logger.info('Requesting details for program {uuid}.'.format(uuid=uuid))
program = client.programs(uuid).get(exclude_utm=1)
# pathways get added in process_pathways
program['pathway_ids'] = []
programs[cache_key] = program
except: # pylint: disable=bare-except
logger.exception('Failed to retrieve details for program {uuid}.'.format(uuid=uuid))
failure = True
continue
return programs, failure
def get_pathways(self, client, site):
"""
Get all pathways for the current client
"""
pathways = {}
failure = False
logger.info('Requesting credit pathways for {domain}.'.format(domain=site.domain))
try:
pathways = client.credit_pathways.get(exclude_utm=1)['results']
except: # pylint: disable=bare-except
logger.error('Failed to retrieve credit pathways for site: {domain}.'.format(domain=site.domain))
failure = True
logger.info('Received {total} credit pathways for site {domain}'.format(
total=len(pathways),
domain=site.domain
))
return pathways, failure
def process_pathways(self, site, pathways, programs):
"""
For each program, add references to each pathway it is a part of.
For each pathway, replace the "programs" dict with "program_uuids",
which only contains uuids (since program data is already cached)
"""
processed_pathways = {}
failure = False
for pathway in pathways:
try:
pathway_id = pathway['id']
pathway_cache_key = CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=pathway_id)
processed_pathways[pathway_cache_key] = pathway
uuids = []
for program in pathway['programs']:
program_uuid = program['uuid']
program_cache_key = PROGRAM_CACHE_KEY_TPL.format(uuid=program_uuid)
programs[program_cache_key]['pathway_ids'].append(pathway_id)
uuids.append(program_uuid)
del pathway['programs']
pathway['program_uuids'] = uuids
except: # pylint: disable=bare-except
logger.error('Failed to process pathways for {domain}'.format(domain=site.domain))
failure = True
return processed_pathways, programs, failure

114
openedx/core/djangoapps/catalog/management/commands/tests/test_cache_programs.py
View File

@@ -5,10 +5,12 @@ from django.core.cache import cache
from django.core.management import call_command
from openedx.core.djangoapps.catalog.cache import (
CREDIT_PATHWAY_CACHE_KEY_TPL,
PROGRAM_CACHE_KEY_TPL,
SITE_CREDIT_PATHWAY_IDS_CACHE_KEY_TPL,
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
)
from openedx.core.djangoapps.catalog.tests.factories import ProgramFactory
from openedx.core.djangoapps.catalog.tests.factories import CreditPathwayFactory, ProgramFactory
from openedx.core.djangoapps.catalog.tests.mixins import CatalogIntegrationMixin
from openedx.core.djangoapps.site_configuration.tests.mixins import SiteMixin
from openedx.core.djangolib.testing.utils import CacheIsolationTestCase, skip_unless_lms
@@ -36,10 +38,20 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
self.list_url = self.catalog_integration.get_internal_api_url().rstrip('/') + '/programs/'
self.detail_tpl = self.list_url.rstrip('/') + '/{uuid}/'
self.pathway_url = self.catalog_integration.get_internal_api_url().rstrip('/') + '/credit_pathways/'
self.programs = ProgramFactory.create_batch(3)
self.pathways = CreditPathwayFactory.create_batch(3)
for pathway in self.pathways:
self.programs += pathway['programs']
self.uuids = [program['uuid'] for program in self.programs]
# add some of the previously created programs to some pathways
self.pathways[0]['programs'].extend([self.programs[0], self.programs[1]])
self.pathways[1]['programs'].append(self.programs[0])
def mock_list(self):
def list_callback(request, uri, headers):
expected = {
@@ -74,7 +86,29 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
content_type='application/json'
)
def test_handle(self):
def mock_pathways(self):
"""
Mock the data for discovery's credit pathways endpoint
"""
def pathways_callback(request, uri, headers): # pylint: disable=unused-argument
"""
Mocks response
"""
expected = {
'exclude_utm': ['1'],
}
self.assertEqual(request.querystring, expected)
return (200, headers, json.dumps({'results': self.pathways}))
httpretty.register_uri(
httpretty.GET,
self.pathway_url,
body=pathways_callback,
content_type='application/json'
)
def test_handle_programs(self):
"""
Verify that the command requests and caches program UUIDs and details.
"""
@@ -89,6 +123,7 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
}
self.mock_list()
self.mock_pathways()
for uuid in self.uuids:
program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
@@ -115,8 +150,58 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
# of the cache above, so all we need to do here is verify the accuracy of
# the data itself.
for key, program in cached_programs.items():
# cached programs have a pathways field added to them, remove before comparing
del program['pathway_ids']
self.assertEqual(program, programs[key])
def test_handle_pathways(self):
"""
Verify that the command requests and caches credit pathways
"""
UserFactory(username=self.catalog_integration.service_username)
programs = {
PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']): program for program in self.programs
}
pathways = {
CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=pathway['id']): pathway for pathway in self.pathways
}
self.mock_list()
self.mock_pathways()
for uuid in self.uuids:
program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
self.mock_detail(uuid, program)
call_command('cache_programs')
cached_pathway_keys = cache.get(SITE_CREDIT_PATHWAY_IDS_CACHE_KEY_TPL.format(domain=self.site_domain))
pathway_keys = pathways.keys()
self.assertEqual(
set(cached_pathway_keys),
set(pathway_keys)
)
cached_pathways = cache.get_many(pathway_keys)
self.assertEqual(
set(cached_pathways),
set(pathways)
)
# We can't use a set comparison here because these values are dictionaries
# and aren't hashable. We've already verified that all pathways came out
# of the cache above, so all we need to do here is verify the accuracy of
# the data itself.
for key, pathway in cached_pathways.items():
# cached pathways store just program uuids instead of the full programs, transform before comparing
pathways[key]['program_uuids'] = [program['uuid'] for program in pathways[key]['programs']]
del pathways[key]['programs']
self.assertEqual(pathway, pathways[key])
def test_handle_missing_service_user(self):
"""
Verify that the command raises an exception when run without a service
@@ -142,6 +227,29 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
self.assertEqual(cached_uuids, [])
def test_handle_missing_pathways(self):
"""
Verify that the command raises an exception when it fails to retrieve pathways.
"""
UserFactory(username=self.catalog_integration.service_username)
programs = {
PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']): program for program in self.programs
}
self.mock_list()
for uuid in self.uuids:
program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
self.mock_detail(uuid, program)
with self.assertRaises(SystemExit) as context:
call_command('cache_programs')
self.assertEqual(context.exception.code, 1)
cached_pathways = cache.get(SITE_CREDIT_PATHWAY_IDS_CACHE_KEY_TPL.format(domain=self.site_domain))
self.assertEqual(cached_pathways, [])
def test_handle_missing_programs(self):
"""
Verify that a problem retrieving a program doesn't prevent the command
@@ -183,4 +291,6 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
)
for key, program in cached_programs.items():
# cached programs have a pathways field added to them, remove before comparing
del program['pathway_ids']
self.assertEqual(program, partial_programs[key])

10
openedx/core/djangoapps/catalog/tests/factories.py
View File

@@ -211,3 +211,13 @@ class ProgramFactory(DictFactoryBase):
class ProgramTypeFactory(DictFactoryBase):
name = factory.Faker('word')
logo_image = factory.LazyFunction(generate_sized_stdimage)
class CreditPathwayFactory(DictFactoryBase):
id = factory.Sequence(lambda x: x)
description = factory.Faker('sentence')
destination_url = factory.Faker('url')
email = factory.Faker('email')
name = factory.Faker('sentence')
org_name = factory.Faker('company')
programs = factory.LazyFunction(partial(generate_instances, ProgramFactory))

140
openedx/core/djangoapps/catalog/tests/test_utils.py
View File

@@ -16,11 +16,17 @@ from course_modes.helpers import CourseMode
from course_modes.tests.factories import CourseModeFactory
from entitlements.tests.factories import CourseEntitlementFactory
from openedx.core.constants import COURSE_UNPUBLISHED
from openedx.core.djangoapps.catalog.cache import PROGRAM_CACHE_KEY_TPL, SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
from openedx.core.djangoapps.catalog.cache import (
CREDIT_PATHWAY_CACHE_KEY_TPL,
PROGRAM_CACHE_KEY_TPL,
SITE_CREDIT_PATHWAY_IDS_CACHE_KEY_TPL,
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
)
from openedx.core.djangoapps.catalog.models import CatalogIntegration
from openedx.core.djangoapps.catalog.tests.factories import (
CourseFactory,
CourseRunFactory,
CreditPathwayFactory,
ProgramFactory,
ProgramTypeFactory
)
@@ -29,6 +35,7 @@ from openedx.core.djangoapps.catalog.utils import (
get_course_runs,
get_course_runs_for_course,
get_course_run_details,
get_credit_pathways,
get_currency_data,
get_localized_price_text,
get_program_types,
@@ -177,6 +184,137 @@ class TestGetPrograms(CacheIsolationTestCase):
self.assertFalse(mock_warning.called)
@skip_unless_lms
@mock.patch(UTILS_MODULE + '.logger.info')
@mock.patch(UTILS_MODULE + '.logger.warning')
class TestGetCreditPathways(CacheIsolationTestCase):
ENABLED_CACHES = ['default']
def setUp(self):
super(TestGetCreditPathways, self).setUp()
self.site = SiteFactory()
def test_get_many(self, mock_warning, mock_info):
pathways = CreditPathwayFactory.create_batch(3)
# Cache details for 2 of 3 programs.
partial_pathways = {
CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=pathway['id']): pathway for pathway in pathways[:2]
}
cache.set_many(partial_pathways, None)
# When called before pathways are cached, the function should return an
# empty list and log a warning.
self.assertEqual(get_credit_pathways(self.site), [])
mock_warning.assert_called_once_with('Failed to get credit pathway ids from the cache.')
mock_warning.reset_mock()
# Cache all 3 pathways
cache.set(
SITE_CREDIT_PATHWAY_IDS_CACHE_KEY_TPL.format(domain=self.site.domain),
[pathway['id'] for pathway in pathways],
None
)
actual_pathways = get_credit_pathways(self.site)
# The 2 cached pathways should be returned while info and warning
# messages should be logged for the missing one.
self.assertEqual(
set(pathway['id'] for pathway in actual_pathways),
set(pathway['id'] for pathway in partial_pathways.values())
)
mock_info.assert_called_with('Failed to get details for 1 pathways. Retrying.')
mock_warning.assert_called_with(
'Failed to get details for credit pathway {id} from the cache.'.format(id=pathways[2]['id'])
)
mock_warning.reset_mock()
# We can't use a set comparison here because these values are dictionaries
# and aren't hashable. We've already verified that all pathways came out
# of the cache above, so all we need to do here is verify the accuracy of
# the data itself.
for pathway in actual_pathways:
key = CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=pathway['id'])
self.assertEqual(pathway, partial_pathways[key])
# Cache details for all 3 pathways.
all_pathways = {
CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=pathway['id']): pathway for pathway in pathways
}
cache.set_many(all_pathways, None)
actual_pathways = get_credit_pathways(self.site)
# All 3 pathways should be returned.
self.assertEqual(
set(pathway['id'] for pathway in actual_pathways),
set(pathway['id'] for pathway in all_pathways.values())
)
self.assertFalse(mock_warning.called)
for pathway in actual_pathways:
key = CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=pathway['id'])
self.assertEqual(pathway, all_pathways[key])
@mock.patch(UTILS_MODULE + '.cache')
def test_get_many_with_missing(self, mock_cache, mock_warning, mock_info):
pathways = CreditPathwayFactory.create_batch(3)
all_pathways = {
CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=pathway['id']): pathway for pathway in pathways
}
partial_pathways = {
CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=pathway['id']): pathway for pathway in pathways[:2]
}
def fake_get_many(keys):
if len(keys) == 1:
return {CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=pathways[-1]['id']): pathways[-1]}
else:
return partial_pathways
mock_cache.get.return_value = [pathway['id'] for pathway in pathways]
mock_cache.get_many.side_effect = fake_get_many
actual_pathways = get_credit_pathways(self.site)
# All 3 cached pathways should be returned. An info message should be
# logged about the one that was initially missing, but the code should
# be able to stitch together all the details.
self.assertEqual(
set(pathway['id'] for pathway in actual_pathways),
set(pathway['id'] for pathway in all_pathways.values())
)
self.assertFalse(mock_warning.called)
mock_info.assert_called_with('Failed to get details for 1 pathways. Retrying.')
for pathway in actual_pathways:
key = CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=pathway['id'])
self.assertEqual(pathway, all_pathways[key])
def test_get_one(self, mock_warning, _mock_info):
expected_pathway = CreditPathwayFactory()
expected_id = expected_pathway['id']
self.assertEqual(get_credit_pathways(self.site, pathway_id=expected_id), None)
mock_warning.assert_called_once_with(
'Failed to get details for credit pathway {id} from the cache.'.format(id=expected_id)
)
mock_warning.reset_mock()
cache.set(
CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=expected_id),
expected_pathway,
None
)
actual_pathway = get_credit_pathways(self.site, pathway_id=expected_id)
self.assertEqual(actual_pathway, expected_pathway)
self.assertFalse(mock_warning.called)
@mock.patch(UTILS_MODULE + '.get_edx_api_data')
class TestGetProgramTypes(CatalogIntegrationMixin, TestCase):
"""Tests covering retrieval of program types from the catalog service."""

59
openedx/core/djangoapps/catalog/utils.py
View File

@@ -14,7 +14,8 @@ from pytz import UTC
from entitlements.utils import is_course_run_entitlement_fulfillable
from openedx.core.constants import COURSE_PUBLISHED
from openedx.core.djangoapps.catalog.cache import (PROGRAM_CACHE_KEY_TPL,
from openedx.core.djangoapps.catalog.cache import (CREDIT_PATHWAY_CACHE_KEY_TPL, PROGRAM_CACHE_KEY_TPL,
SITE_CREDIT_PATHWAY_IDS_CACHE_KEY_TPL,
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL)
from openedx.core.djangoapps.catalog.models import CatalogIntegration
from openedx.core.lib.edx_api_utils import get_edx_api_data
@@ -124,6 +125,62 @@ def get_program_types(name=None):
return []
def get_credit_pathways(site, pathway_id=None):
"""
Read pathways from the cache.
The cache is populated by a management command, cache_programs.
Arguments:
site (Site): django.contrib.sites.models object
Keyword Arguments:
pathway_id (string): id identifying a specific pathway to read from the cache.
Returns:
list of dict, representing pathways.
dict, if a specific pathway is requested.
"""
missing_details_msg_tpl = 'Failed to get details for credit pathway {id} from the cache.'
if pathway_id:
pathway = cache.get(CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=pathway_id))
if not pathway:
logger.warning(missing_details_msg_tpl.format(id=pathway_id))
return pathway
pathway_ids = cache.get(SITE_CREDIT_PATHWAY_IDS_CACHE_KEY_TPL.format(domain=site.domain), [])
if not pathway_ids:
logger.warning('Failed to get credit pathway ids from the cache.')
pathways = cache.get_many([CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=pathway_id) for pathway_id in pathway_ids])
pathways = pathways.values()
# The get_many above sometimes fails to bring back details cached on one or
# more Memcached nodes. It doesn't look like these keys are being evicted.
# 99% of the time all keys come back, but 1% of the time all the keys stored
# on one or more nodes are missing from the result of the get_many. One
# get_many may fail to bring these keys back, but a get_many occurring
# immediately afterwards will succeed in bringing back all the keys. This
# behavior can be mitigated by trying again for the missing keys, which is
# what we do here. Splitting the get_many into smaller chunks may also help.
missing_ids = set(pathway_ids) - set(pathway['id'] for pathway in pathways)
if missing_ids:
logger.info(
'Failed to get details for {count} pathways. Retrying.'.format(count=len(missing_ids))
)
retried_pathways = cache.get_many(
[CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=pathway_id) for pathway_id in missing_ids]
)
pathways += retried_pathways.values()
still_missing_ids = set(pathway_ids) - set(pathway['id'] for pathway in pathways)
for missing_id in still_missing_ids:
logger.warning(missing_details_msg_tpl.format(id=missing_id))
return pathways
def get_currency_data():
"""Retrieve currency data from the catalog service.