Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

multitenant Program cache.

Browse Source 
Fetched Programs and Program details from Course Discovery service for all sites and stored the uuids in cache with site-specfic keys.
Learner-1146
This commit is contained in:
Afzal Wali
2017-06-14 15:13:16 +05:00
parent 51fc7b73bf
commit 383208c4c8
11 changed files with 205 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
openedx/core/djangoapps/catalog/cache.py
View File

@@ -2,4 +2,9 @@
PROGRAM_CACHE_KEY_TPL = 'program-{uuid}'
# Cache key used to locate an item containing a list of all program UUIDs.
# This has to be deleted when removing the waffle flags populate-multitenant-programs and get-multitenant-programs
# For more, see LEARNER-1146
PROGRAM_UUIDS_CACHE_KEY = 'program-uuids'
# Cache key used to locate an item containing a list of all program UUIDs for a site.
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL = 'program-uuids-{domain}'

152
openedx/core/djangoapps/catalog/management/commands/cache_programs.py
View File

@@ -1,11 +1,17 @@
import logging
import sys
import waffle
from django.contrib.auth import get_user_model
from django.contrib.sites.models import Site
from django.core.cache import cache
from django.core.management import BaseCommand
from openedx.core.djangoapps.catalog.cache import PROGRAM_CACHE_KEY_TPL, PROGRAM_UUIDS_CACHE_KEY
from openedx.core.djangoapps.catalog.cache import (
PROGRAM_CACHE_KEY_TPL,
PROGRAM_UUIDS_CACHE_KEY,
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
)
from openedx.core.djangoapps.catalog.models import CatalogIntegration
from openedx.core.djangoapps.catalog.utils import create_catalog_api_client
@@ -24,18 +30,111 @@ class Command(BaseCommand):
help = "Rebuild the LMS' cache of program data."
def handle(self, *args, **options):
catalog_integration = CatalogIntegration.current()
username = catalog_integration.service_username
if waffle.switch_is_active('populate-multitenant-programs'):
failure = False
logger.info('populate-multitenant-programs switch is ON')
try:
user = User.objects.get(username=username)
client = create_catalog_api_client(user)
except User.DoesNotExist:
logger.error(
'Failed to create API client. Service user {username} does not exist.'.format(username)
)
raise
catalog_integration = CatalogIntegration.current()
username = catalog_integration.service_username
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
logger.error(
'Failed to create API client. Service user {username} does not exist.'.format(username)
)
raise
programs = {}
for site in Site.objects.all():
site_config = getattr(site, 'configuration', None)
if site_config is None or not site_config.get_value('COURSE_CATALOG_API_URL'):
logger.info('Skipping site {domain}. No configuration.'.format(domain=site.domain))
continue
client = create_catalog_api_client(user, site=site)
uuids, program_uuids_failed = self.get_site_program_uuids(client, site)
new_programs, program_details_failed = self.fetch_program_details(client, uuids)
if program_uuids_failed or program_details_failed:
failure = True
programs.update(new_programs)
logger.info('Caching UUIDs for {total} programs for site {site_name}.'.format(
total=len(uuids),
site_name=site.domain,
))
cache.set(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), uuids, None)
successful = len(programs)
logger.info('Caching details for {successful} programs.'.format(successful=successful))
cache.set_many(programs, None)
if failure:
# This will fail a Jenkins job running this command, letting site
# operators know that there was a problem.
sys.exit(1)
else:
catalog_integration = CatalogIntegration.current()
username = catalog_integration.service_username
try:
user = User.objects.get(username=username)
client = create_catalog_api_client(user)
except User.DoesNotExist:
logger.error(
'Failed to create API client. Service user {username} does not exist.'.format(username)
)
raise
try:
querystring = {
'exclude_utm': 1,
'status': ('active', 'retired'),
'uuids_only': 1,
}
logger.info('Requesting program UUIDs.')
uuids = client.programs.get(**querystring)
except: # pylint: disable=bare-except
logger.error('Failed to retrieve program UUIDs.')
raise
total = len(uuids)
logger.info('Received {total} UUIDs.'.format(total=total))
programs = {}
failure = False
for uuid in uuids:
try:
logger.info('Requesting details for program {uuid}.'.format(uuid=uuid))
program = client.programs(uuid).get(exclude_utm=1)
cache_key = PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)
programs[cache_key] = program
except: # pylint: disable=bare-except
logger.exception('Failed to retrieve details for program {uuid}.'.format(uuid=uuid))
failure = True
continue
successful = len(programs)
logger.info('Caching details for {successful} programs.'.format(successful=successful))
cache.set_many(programs, None)
logger.info('Caching UUIDs for {total} programs.'.format(total=total))
cache.set(PROGRAM_UUIDS_CACHE_KEY, uuids, None)
if failure:
# This will fail a Jenkins job running this command, letting site
# operators know that there was a problem.
sys.exit(1)
def get_site_program_uuids(self, client, site):
failure = False
uuids = []
try:
querystring = {
'exclude_utm': 1,
@@ -43,38 +142,29 @@ class Command(BaseCommand):
'uuids_only': 1,
}
logger.info('Requesting program UUIDs.')
logger.info('Requesting program UUIDs for {domain}.'.format(domain=site.domain))
uuids = client.programs.get(**querystring)
except: # pylint: disable=bare-except
logger.error('Failed to retrieve program UUIDs.')
raise
logger.error('Failed to retrieve program UUIDs for site: {domain}.'.format(domain=site.domain))
failure = True
total = len(uuids)
logger.info('Received {total} UUIDs.'.format(total=total))
logger.info('Received {total} UUIDs for site {domain}'.format(
total=len(uuids),
domain=site.domain
))
return uuids, failure
def fetch_program_details(self, client, uuids):
programs = {}
failure = False
for uuid in uuids:
try:
cache_key = PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)
logger.info('Requesting details for program {uuid}.'.format(uuid=uuid))
program = client.programs(uuid).get(exclude_utm=1)
cache_key = PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)
programs[cache_key] = program
except: # pylint: disable=bare-except
logger.exception('Failed to retrieve details for program {uuid}.'.format(uuid=uuid))
failure = True
continue
successful = len(programs)
logger.info('Caching details for {successful} programs.'.format(successful=successful))
cache.set_many(programs, None)
logger.info('Caching UUIDs for {total} programs.'.format(total=total))
cache.set(PROGRAM_UUIDS_CACHE_KEY, uuids, None)
if failure:
# This will fail a Jenkins job running this command, letting site
# operators know that there was a problem.
sys.exit(1)
return programs, failure

34
openedx/core/djangoapps/catalog/management/commands/tests/test_cache_programs.py
View File

@@ -1,25 +1,38 @@
import json
import httpretty
import waffle
from django.core.cache import cache
from django.core.management import call_command
from openedx.core.djangoapps.catalog.cache import PROGRAM_CACHE_KEY_TPL, PROGRAM_UUIDS_CACHE_KEY
from openedx.core.djangoapps.catalog.cache import (
PROGRAM_CACHE_KEY_TPL,
PROGRAM_UUIDS_CACHE_KEY,
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
)
from openedx.core.djangoapps.catalog.tests.factories import ProgramFactory
from openedx.core.djangoapps.catalog.tests.mixins import CatalogIntegrationMixin
from openedx.core.djangoapps.site_configuration.tests.mixins import SiteMixin
from openedx.core.djangolib.testing.utils import CacheIsolationTestCase, skip_unless_lms
from student.tests.factories import UserFactory
@skip_unless_lms
@httpretty.activate
class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMixin):
ENABLED_CACHES = ['default']
def setUp(self):
super(TestCachePrograms, self).setUp()
self.catalog_integration = self.create_catalog_integration()
self.site_domain = 'testsite.com'
self.set_up_site(
self.site_domain,
{
'COURSE_CATALOG_API_URL': self.catalog_integration.get_internal_api_url().rstrip('/')
}
)
self.list_url = self.catalog_integration.get_internal_api_url().rstrip('/') + '/programs/'
self.detail_tpl = self.list_url.rstrip('/') + '/{uuid}/'
@@ -61,6 +74,7 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
content_type='application/json'
)
@waffle.testutils.override_switch('populate-multitenant-programs', True)
def test_handle(self):
"""
Verify that the command requests and caches program UUIDs and details.
@@ -83,7 +97,7 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
call_command('cache_programs')
cached_uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY)
cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
self.assertEqual(
set(cached_uuids),
set(self.uuids)
@@ -104,6 +118,7 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
for key, program in cached_programs.items():
self.assertEqual(program, programs[key])
@waffle.testutils.override_switch('populate-multitenant-programs', True)
def test_handle_missing_service_user(self):
"""
Verify that the command raises an exception when run without a service
@@ -112,9 +127,10 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
with self.assertRaises(Exception):
call_command('cache_programs')
cached_uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY)
cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
self.assertEqual(cached_uuids, None)
@waffle.testutils.override_switch('populate-multitenant-programs', True)
def test_handle_missing_uuids(self):
"""
Verify that the command raises an exception when it fails to retrieve
@@ -122,12 +138,14 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
"""
UserFactory(username=self.catalog_integration.service_username)
with self.assertRaises(Exception):
with self.assertRaises(SystemExit) as context:
call_command('cache_programs')
self.assertEqual(context.exception.code, 1)
cached_uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY)
self.assertEqual(cached_uuids, None)
cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
self.assertEqual(cached_uuids, [])
@waffle.testutils.override_switch('populate-multitenant-programs', True)
def test_handle_missing_programs(self):
"""
Verify that a problem retrieving a program doesn't prevent the command
@@ -154,7 +172,7 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
self.assertEqual(context.exception.code, 1)
cached_uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY)
cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
self.assertEqual(
set(cached_uuids),
set(self.uuids)

37
openedx/core/djangoapps/catalog/utils.py
View File

@@ -2,26 +2,36 @@
import copy
import logging
import waffle
from django.conf import settings
from django.core.cache import cache
from django.core.exceptions import ObjectDoesNotExist
from edx_rest_api_client.client import EdxRestApiClient
from openedx.core.djangoapps.catalog.cache import PROGRAM_CACHE_KEY_TPL, PROGRAM_UUIDS_CACHE_KEY
from openedx.core.djangoapps.catalog.cache import (
PROGRAM_CACHE_KEY_TPL,
PROGRAM_UUIDS_CACHE_KEY,
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
)
from openedx.core.djangoapps.catalog.models import CatalogIntegration
from openedx.core.djangoapps.theming.helpers import get_current_site
from openedx.core.lib.edx_api_utils import get_edx_api_data
from openedx.core.lib.token_utils import JwtBuilder
logger = logging.getLogger(__name__)
def create_catalog_api_client(user):
def create_catalog_api_client(user, site=None):
"""Returns an API client which can be used to make Catalog API requests."""
scopes = ['email', 'profile']
expires_in = settings.OAUTH_ID_TOKEN_EXPIRATION
jwt = JwtBuilder(user).build_token(scopes, expires_in)
url = CatalogIntegration.current().get_internal_api_url()
if site:
url = site.configuration.get_value('COURSE_CATALOG_API_URL')
else:
url = CatalogIntegration.current().get_internal_api_url()
return EdxRestApiClient(url, jwt=jwt)
@@ -45,8 +55,10 @@ def get_programs(uuid=None):
logger.warning(missing_details_msg_tpl.format(uuid=uuid))
return program
uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY, [])
if waffle.switch_is_active('get-multitenant-programs'):
uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=get_current_site().domain), [])
else:
uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY, [])
if not uuids:
logger.warning('Failed to get program UUIDs from the cache.')
@@ -109,16 +121,14 @@ def get_program_types(name=None):
return []
def get_programs_with_type(types=None, include_hidden=True):
def get_programs_with_type(include_hidden=True):
"""
Return the list of programs. You can filter the types of programs returned using the optional
types parameter. If no filter is provided, all programs of all types will be returned. In addition,
you can specify whether to include hidden programs using the optional include_hidden parameter.
Return the list of programs. You can filter the types of programs returned by using the optional
include_hidden parameter. By default hidden programs will be included.
The program dict is updated with the fully serialized program type.
Keyword Arguments:
types (list): List of program type slugs to filter by.
include_hidden (bool): whether to include hidden programs
Return:
@@ -130,12 +140,7 @@ def get_programs_with_type(types=None, include_hidden=True):
if programs:
program_types = {program_type['name']: program_type for program_type in get_program_types()}
for program in programs:
# This limited type filtering is sufficient for current needs and
# helps us avoid additional complexity when caching program data.
# However, if the need for additional filtering of programs should
# arise, consider using the cache_programs management command to
# cache the filtered lists of UUIDs.
if types and program['type'] not in types:
if program['type'] not in program_types:
continue
if program['hidden'] and not include_hidden: