Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reset integration environment endpoint (#21319)

Browse Source
This commit is contained in:
Zachary Hancock
2019-08-14 13:19:45 -04:00
committed by GitHub
parent 4829fd4fde
commit 96f99bc053
11 changed files with 287 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
openedx/core/djangoapps/catalog/cache.py
View File

@@ -18,3 +18,6 @@ COURSE_PROGRAMS_CACHE_KEY_TPL = 'course-programs-{course_run_id}'
# because program_type values are likely to be shared between different sites
# that live in the same environment).
PROGRAMS_BY_TYPE_CACHE_KEY_TPL = 'programs-by-type-{site_id}-{program_type}'
# Template used to create cache keys for organization to program uuids.
PROGRAMS_BY_ORGANIZATION_CACHE_KEY_TPL = 'organization-programs-{org_key}'

26
openedx/core/djangoapps/catalog/management/commands/cache_programs.py
View File

@@ -1,8 +1,9 @@
""""Management command to add program information to the cache."""
from __future__ import absolute_import
from collections import defaultdict
import logging
import sys
from collections import defaultdict
from django.contrib.auth import get_user_model
from django.contrib.sites.models import Site
@@ -14,15 +15,16 @@ from openedx.core.djangoapps.catalog.cache import (
COURSE_PROGRAMS_CACHE_KEY_TPL,
PATHWAY_CACHE_KEY_TPL,
PROGRAM_CACHE_KEY_TPL,
PROGRAMS_BY_ORGANIZATION_CACHE_KEY_TPL,
PROGRAMS_BY_TYPE_CACHE_KEY_TPL,
SITE_PATHWAY_IDS_CACHE_KEY_TPL,
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL,
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
)
from openedx.core.djangoapps.catalog.models import CatalogIntegration
from openedx.core.djangoapps.catalog.utils import (
create_catalog_api_client,
course_run_keys_for_program,
normalize_program_type,
create_catalog_api_client,
normalize_program_type
)
logger = logging.getLogger(__name__)
@@ -59,6 +61,7 @@ class Command(BaseCommand):
pathways = {}
courses = {}
programs_by_type = {}
organizations = {}
for site in Site.objects.all():
site_config = getattr(site, 'configuration', None)
if site_config is None or not site_config.get_value('COURSE_CATALOG_API_URL'):
@@ -86,6 +89,7 @@ class Command(BaseCommand):
pathways.update(new_pathways)
courses.update(self.get_courses(new_programs))
programs_by_type.update(self.get_programs_by_type(site, new_programs))
organizations.update(self.get_programs_by_organization(new_programs))
logger.info(u'Caching UUIDs for {total} programs for site {site_name}.'.format(
total=len(uuids),
@@ -112,6 +116,9 @@ class Command(BaseCommand):
logger.info(text_type('Caching program UUIDs by {} program types.'.format(len(programs_by_type))))
cache.set_many(programs_by_type, None)
logger.info(u'Caching programs uuids for {} organizations'.format(len(organizations)))
cache.set_many(organizations, None)
if failure:
sys.exit(1)
@@ -236,3 +243,14 @@ class Command(BaseCommand):
cache_key = PROGRAMS_BY_TYPE_CACHE_KEY_TPL.format(site_id=site.id, program_type=program_type)
programs_by_type[cache_key].append(program['uuid'])
return programs_by_type
def get_programs_by_organization(self, programs):
"""
Returns a dictionary mapping organization keys to lists of program uuids authored by that org
"""
organizations = defaultdict(list)
for program in programs.values():
for org in program['authoring_organizations']:
org_cache_key = PROGRAMS_BY_ORGANIZATION_CACHE_KEY_TPL.format(org_key=org['key'])
organizations[org_cache_key].append(program['uuid'])
return organizations

13
openedx/core/djangoapps/catalog/management/commands/tests/test_cache_programs.py
View File

@@ -10,6 +10,7 @@ from django.core.management import call_command
from openedx.core.djangoapps.catalog.cache import (
COURSE_PROGRAMS_CACHE_KEY_TPL,
PROGRAMS_BY_ORGANIZATION_CACHE_KEY_TPL,
PATHWAY_CACHE_KEY_TPL,
PROGRAM_CACHE_KEY_TPL,
PROGRAMS_BY_TYPE_CACHE_KEY_TPL,
@@ -17,7 +18,7 @@ from openedx.core.djangoapps.catalog.cache import (
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
)
from openedx.core.djangoapps.catalog.utils import normalize_program_type
from openedx.core.djangoapps.catalog.tests.factories import PathwayFactory, ProgramFactory
from openedx.core.djangoapps.catalog.tests.factories import OrganizationFactory, PathwayFactory, ProgramFactory
from openedx.core.djangoapps.catalog.tests.mixins import CatalogIntegrationMixin
from openedx.core.djangoapps.site_configuration.tests.mixins import SiteMixin
from openedx.core.djangolib.testing.utils import CacheIsolationTestCase, skip_unless_lms
@@ -57,6 +58,8 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
self.programs[0]['curricula'][0]['programs'].append(self.child_program)
self.programs.append(self.child_program)
self.programs[0]['authoring_organizations'] = OrganizationFactory.create_batch(2)
for pathway in self.pathways:
self.programs += pathway['programs']
@@ -193,7 +196,7 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
self.assertIn(self.child_program['uuid'], cache.get(course_run_cache_key))
# for each program, assert that the program's UUID is in a cached list of
# program UUIDS by program type
# program UUIDS by program type and a cached list of UUIDs by authoring organization
for program in self.programs:
program_type = normalize_program_type(program.get('type', 'None'))
program_type_cache_key = PROGRAMS_BY_TYPE_CACHE_KEY_TPL.format(
@@ -201,6 +204,12 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMix
)
self.assertIn(program['uuid'], cache.get(program_type_cache_key))
for organization in program['authoring_organizations']:
organization_cache_key = PROGRAMS_BY_ORGANIZATION_CACHE_KEY_TPL.format(
org_key=organization['key']
)
self.assertIn(program['uuid'], cache.get(organization_cache_key))
def test_handle_pathways(self):
"""
Verify that the command requests and caches credit pathways

17
openedx/core/djangoapps/catalog/utils.py
View File

@@ -18,6 +18,7 @@ from entitlements.utils import is_course_run_entitlement_fulfillable
from openedx.core.constants import COURSE_PUBLISHED
from openedx.core.djangoapps.catalog.cache import (
COURSE_PROGRAMS_CACHE_KEY_TPL,
PROGRAMS_BY_ORGANIZATION_CACHE_KEY_TPL,
PATHWAY_CACHE_KEY_TPL,
PROGRAM_CACHE_KEY_TPL,
PROGRAMS_BY_TYPE_CACHE_KEY_TPL,
@@ -86,7 +87,7 @@ def check_catalog_integration_and_get_user(error_message_field):
# pylint: disable=redefined-outer-name
def get_programs(site=None, uuid=None, uuids=None, course=None):
def get_programs(site=None, uuid=None, uuids=None, course=None, organization=None):
"""Read programs from the cache.
The cache is populated by a management command, cache_programs.
@@ -96,12 +97,13 @@ def get_programs(site=None, uuid=None, uuids=None, course=None):
uuid (string): UUID identifying a specific program to read from the cache.
uuids (list of string): UUIDs identifying a specific programs to read from the cache.
course (string): course id identifying a specific course run to read from the cache.
organization (string): short name for specific organization to read from the cache.
Returns:
list of dict, representing programs.
dict, if a specific program is requested.
"""
if len([arg for arg in (site, uuid, uuids, course) if arg is not None]) != 1:
if len([arg for arg in (site, uuid, uuids, course, organization) if arg is not None]) != 1:
raise TypeError('get_programs takes exactly one argument')
if uuid:
@@ -120,6 +122,10 @@ def get_programs(site=None, uuid=None, uuids=None, course=None):
uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), [])
if not uuids:
logger.warning(u'Failed to get program UUIDs from the cache for site {}.'.format(site.domain))
elif organization:
uuids = get_programs_for_organization(organization)
if not uuids:
return []
return get_programs_by_uuids(uuids)
@@ -623,3 +629,10 @@ def _course_runs_from_container(container):
def normalize_program_type(program_type):
""" Function that normalizes a program type string for use in a cache key. """
return str(program_type).lower()
def get_programs_for_organization(organization):
"""
Retrieve list of program uuids authored by a given organization
"""
return cache.get(PROGRAMS_BY_ORGANIZATION_CACHE_KEY_TPL.format(org_key=organization))