Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #15334 from edx/afzaledx/Learner-1146

Browse Source 
Learner-1146
This commit is contained in:
Afzal Wali Naushahi
2017-07-04 15:17:38 +05:00
committed by GitHub
parent 51fc7b73bf 383208c4c8
commit f4da106e36
11 changed files with 205 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
common/djangoapps/student/views.py
View File

@@ -36,7 +36,6 @@ from django.utils.translation import get_language, ungettext
from django.views.decorators.csrf import csrf_exempt, ensure_csrf_cookie
from django.views.decorators.http import require_GET, require_POST
from django.views.generic import TemplateView
from eventtracking import tracker
from ipware.ip import get_ip
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
@@ -46,9 +45,9 @@ from provider.oauth2.models import Client
from pytz import UTC
from ratelimitbackend.exceptions import RateLimitException
from requests import HTTPError
from social_django import utils as social_utils
from social_core.backends import oauth as social_oauth
from social_core.exceptions import AuthAlreadyAssociated, AuthException
from social_django import utils as social_utils
import dogstats_wrapper as dog_stats_api
import openedx.core.djangoapps.external_auth.views
@@ -66,6 +65,7 @@ from courseware.access import has_access
from courseware.courses import get_courses, sort_by_announcement, sort_by_start_date # pylint: disable=import-error
from django_comment_common.models import assign_role
from edxmako.shortcuts import render_to_response, render_to_string
from eventtracking import tracker
from lms.djangoapps.commerce.utils import EcommerceService # pylint: disable=import-error
from lms.djangoapps.grades.new.course_grade_factory import CourseGradeFactory
from lms.djangoapps.verify_student.models import SoftwareSecurePhotoVerification # pylint: disable=import-error
@@ -174,7 +174,6 @@ def index(request, extra_context=None, user=AnonymousUser()):
if extra_context is None:
extra_context = {}
programs_list = []
courses = get_courses(user)
if configuration_helpers.get_value(
@@ -208,17 +207,7 @@ def index(request, extra_context=None, user=AnonymousUser()):
# Insert additional context for use in the template
context.update(extra_context)
# Get the active programs of the type configured for the current site from the catalog service. The programs_list
# is being added to the context but it's not being used currently in courseware/courses.html. To use this list,
# you need to create a custom theme that overrides courses.html. The modifications to courses.html to display the
# programs will be done after the support for edx-pattern-library is added.
program_types = configuration_helpers.get_value('ENABLED_PROGRAM_TYPES')
# Do not add programs to the context if there are no program types enabled for the site.
if program_types:
programs_list = get_programs_with_type(program_types, include_hidden=False)
context["programs_list"] = programs_list
context['programs_list'] = get_programs_with_type(include_hidden=False)
return render_to_response('index.html', context)

5
common/djangoapps/terrain/stubs/catalog.py
View File

@@ -14,6 +14,7 @@ class StubCatalogServiceHandler(StubHttpRequestHandler):
pattern_handlers = {
r'/api/v1/programs/$': self.program_list,
r'/api/v1/programs/([0-9a-f-]+)/$': self.program_detail,
r'/api/v1/program_types/$': self.program_types,
}
if self.match_pattern(pattern_handlers):
@@ -42,6 +43,10 @@ class StubCatalogServiceHandler(StubHttpRequestHandler):
program = self.server.config.get('catalog.programs.' + program_uuid)
self.send_json_response(program)
def program_types(self):
program_types = self.server.config.get('catalog.programs_types', [])
self.send_json_response(program_types)
class StubCatalogService(StubHttpService):
HANDLER_CLASS = StubCatalogServiceHandler

13
common/test/acceptance/fixtures/catalog.py
View File

@@ -29,7 +29,6 @@ class CatalogFixture(object):
uuids.append(uuid)
program_key = '{base}.{uuid}'.format(base=key, uuid=uuid)
requests.put(
'{}/set_config'.format(CATALOG_STUB_URL),
data={program_key: json.dumps(program)},
@@ -41,6 +40,18 @@ class CatalogFixture(object):
data={key: json.dumps(uuids)},
)
def install_program_types(self, program_types):
"""
Stub the discovery service's program type list API endpoints.
Arguments:
program_types (list): A list of program types. List endpoint will be stubbed using data from this list.
"""
requests.put(
'{}/set_config'.format(CATALOG_STUB_URL),
data={'catalog.programs_types': json.dumps(program_types)},
)
class CatalogIntegrationMixin(object):
"""Mixin providing a method used to configure the catalog integration."""

13
common/test/acceptance/tests/lms/test_programs.py
View File

@@ -8,7 +8,12 @@ from common.test.acceptance.pages.common.auto_auth import AutoAuthPage
from common.test.acceptance.pages.lms.catalog import CacheProgramsPage
from common.test.acceptance.pages.lms.programs import ProgramDetailsPage, ProgramListingPage
from common.test.acceptance.tests.helpers import UniqueCourseTest
from openedx.core.djangoapps.catalog.tests.factories import CourseFactory, CourseRunFactory, ProgramFactory
from openedx.core.djangoapps.catalog.tests.factories import (
CourseFactory,
CourseRunFactory,
ProgramFactory,
ProgramTypeFactory
)
class ProgramPageBase(ProgramsConfigMixin, CatalogIntegrationMixin, UniqueCourseTest):
@@ -36,7 +41,8 @@ class ProgramPageBase(ProgramsConfigMixin, CatalogIntegrationMixin, UniqueCourse
course_run = CourseRunFactory(key=self.course_id)
course = CourseFactory(course_runs=[course_run])
return ProgramFactory(courses=[course])
program_type = ProgramTypeFactory()
return ProgramFactory(courses=[course], type=program_type['name'])
def stub_catalog_api(self, programs):
"""
@@ -45,6 +51,9 @@ class ProgramPageBase(ProgramsConfigMixin, CatalogIntegrationMixin, UniqueCourse
self.set_catalog_integration(is_enabled=True, service_username=self.username)
CatalogFixture().install_programs(programs)
program_types = [program['type'] for program in programs]
CatalogFixture().install_program_types(program_types)
def cache_programs(self):
"""
Populate the LMS' cache of program data.

5
lms/djangoapps/branding/tests/test_page.py
View File

@@ -299,11 +299,6 @@ class IndexPageProgramsTests(SiteMixin, ModuleStoreTestCase):
"""
@ddt.data([], ['fake_program_type'])
def test_get_programs_with_type_called(self, program_types):
self.site_configuration.values.update({
'ENABLED_PROGRAM_TYPES': program_types
})
self.site_configuration.save()
views = [
(reverse('root'), 'student.views.get_programs_with_type'),
(reverse('branding.views.courses'), 'courseware.views.views.get_programs_with_type'),

9
lms/djangoapps/branding/tests/test_views.py
View File

@@ -318,12 +318,3 @@ class TestIndex(SiteMixin, TestCase):
self.client.login(username=self.user.username, password="password")
response = self.client.get(reverse("dashboard"))
self.assertIn(self.site_configuration_other.values["MKTG_URLS"]["ROOT"], response.content)
def test_index_with_enabled_program_types(self):
""" Test index view with Enabled Program Types."""
self.site_configuration.values.update({'ENABLED_PROGRAM_TYPES': ['TestProgramType']})
self.site_configuration.save()
with mock.patch('student.views.get_programs_with_type') as patched_get_programs_with_type:
patched_get_programs_with_type.return_value = []
response = self.client.get(reverse("root"))
self.assertEqual(response.status_code, 200)

10
lms/djangoapps/courseware/views/views.py
View File

@@ -155,15 +155,7 @@ def courses(request):
else:
courses_list = sort_by_announcement(courses_list)
# Get the active programs of the type configured for the current site from the catalog service. The programs_list
# is being added to the context but it's not being used currently in courseware/courses.html. To use this list,
# you need to create a custom theme that overrides courses.html. The modifications to courses.html to display the
# programs will be done after the support for edx-pattern-library is added.
program_types = configuration_helpers.get_value('ENABLED_PROGRAM_TYPES')
# Do not add programs to the context if there are no program types enabled for the site.
if program_types:
programs_list = get_programs_with_type(program_types, include_hidden=False)
programs_list = get_programs_with_type(include_hidden=False)
return render_to_response(
"courseware/courses.html",

5
openedx/core/djangoapps/catalog/cache.py
View File

@@ -2,4 +2,9 @@
PROGRAM_CACHE_KEY_TPL = 'program-{uuid}'
# Cache key used to locate an item containing a list of all program UUIDs.
# This has to be deleted when removing the waffle flags populate-multitenant-programs and get-multitenant-programs
# For more, see LEARNER-1146
PROGRAM_UUIDS_CACHE_KEY = 'program-uuids'
# Cache key used to locate an item containing a list of all program UUIDs for a site.
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL = 'program-uuids-{domain}'

152
openedx/core/djangoapps/catalog/management/commands/cache_programs.py
View File

@@ -1,11 +1,17 @@
import logging
import sys
import waffle
from django.contrib.auth import get_user_model
from django.contrib.sites.models import Site
from django.core.cache import cache
from django.core.management import BaseCommand
from openedx.core.djangoapps.catalog.cache import PROGRAM_CACHE_KEY_TPL, PROGRAM_UUIDS_CACHE_KEY
from openedx.core.djangoapps.catalog.cache import (
PROGRAM_CACHE_KEY_TPL,
PROGRAM_UUIDS_CACHE_KEY,
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
)
from openedx.core.djangoapps.catalog.models import CatalogIntegration
from openedx.core.djangoapps.catalog.utils import create_catalog_api_client
@@ -24,18 +30,111 @@ class Command(BaseCommand):
help = "Rebuild the LMS' cache of program data."
def handle(self, *args, **options):
catalog_integration = CatalogIntegration.current()
username = catalog_integration.service_username
if waffle.switch_is_active('populate-multitenant-programs'):
failure = False
logger.info('populate-multitenant-programs switch is ON')
try:
user = User.objects.get(username=username)
client = create_catalog_api_client(user)
except User.DoesNotExist:
logger.error(
'Failed to create API client. Service user {username} does not exist.'.format(username)
)
raise
catalog_integration = CatalogIntegration.current()
username = catalog_integration.service_username
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
logger.error(
'Failed to create API client. Service user {username} does not exist.'.format(username)
)
raise
programs = {}
for site in Site.objects.all():
site_config = getattr(site, 'configuration', None)
if site_config is None or not site_config.get_value('COURSE_CATALOG_API_URL'):
logger.info('Skipping site {domain}. No configuration.'.format(domain=site.domain))
continue
client = create_catalog_api_client(user, site=site)
uuids, program_uuids_failed = self.get_site_program_uuids(client, site)
new_programs, program_details_failed = self.fetch_program_details(client, uuids)
if program_uuids_failed or program_details_failed:
failure = True
programs.update(new_programs)
logger.info('Caching UUIDs for {total} programs for site {site_name}.'.format(
total=len(uuids),
site_name=site.domain,
))
cache.set(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), uuids, None)
successful = len(programs)
logger.info('Caching details for {successful} programs.'.format(successful=successful))
cache.set_many(programs, None)
if failure:
# This will fail a Jenkins job running this command, letting site
# operators know that there was a problem.
sys.exit(1)
else:
catalog_integration = CatalogIntegration.current()
username = catalog_integration.service_username
try:
user = User.objects.get(username=username)
client = create_catalog_api_client(user)
except User.DoesNotExist:
logger.error(
'Failed to create API client. Service user {username} does not exist.'.format(username)
)
raise
try:
querystring = {
'exclude_utm': 1,
'status': ('active', 'retired'),
'uuids_only': 1,
}
logger.info('Requesting program UUIDs.')
uuids = client.programs.get(**querystring)
except: # pylint: disable=bare-except
logger.error('Failed to retrieve program UUIDs.')
raise
total = len(uuids)
logger.info('Received {total} UUIDs.'.format(total=total))
programs = {}
failure = False
for uuid in uuids:
try:
logger.info('Requesting details for program {uuid}.'.format(uuid=uuid))
program = client.programs(uuid).get(exclude_utm=1)
cache_key = PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)
programs[cache_key] = program
except: # pylint: disable=bare-except
logger.exception('Failed to retrieve details for program {uuid}.'.format(uuid=uuid))
failure = True
continue
successful = len(programs)
logger.info('Caching details for {successful} programs.'.format(successful=successful))
cache.set_many(programs, None)
logger.info('Caching UUIDs for {total} programs.'.format(total=total))
cache.set(PROGRAM_UUIDS_CACHE_KEY, uuids, None)
if failure:
# This will fail a Jenkins job running this command, letting site
# operators know that there was a problem.
sys.exit(1)
def get_site_program_uuids(self, client, site):
failure = False
uuids = []
try:
querystring = {
'exclude_utm': 1,
@@ -43,38 +142,29 @@ class Command(BaseCommand):
'uuids_only': 1,
}
logger.info('Requesting program UUIDs.')
logger.info('Requesting program UUIDs for {domain}.'.format(domain=site.domain))
uuids = client.programs.get(**querystring)
except: # pylint: disable=bare-except
logger.error('Failed to retrieve program UUIDs.')
raise
logger.error('Failed to retrieve program UUIDs for site: {domain}.'.format(domain=site.domain))
failure = True
total = len(uuids)
logger.info('Received {total} UUIDs.'.format(total=total))
logger.info('Received {total} UUIDs for site {domain}'.format(
total=len(uuids),
domain=site.domain
))
return uuids, failure
def fetch_program_details(self, client, uuids):
programs = {}
failure = False
for uuid in uuids:
try:
cache_key = PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)
logger.info('Requesting details for program {uuid}.'.format(uuid=uuid))
program = client.programs(uuid).get(exclude_utm=1)
cache_key = PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)
programs[cache_key] = program
except: # pylint: disable=bare-except
logger.exception('Failed to retrieve details for program {uuid}.'.format(uuid=uuid))
failure = True
continue
successful = len(programs)
logger.info('Caching details for {successful} programs.'.format(successful=successful))
cache.set_many(programs, None)
logger.info('Caching UUIDs for {total} programs.'.format(total=total))
cache.set(PROGRAM_UUIDS_CACHE_KEY, uuids, None)
if failure:
# This will fail a Jenkins job running this command, letting site
# operators know that there was a problem.
sys.exit(1)
return programs, failure

34
openedx/core/djangoapps/catalog/management/commands/tests/test_cache_programs.py
View File

@@ -1,25 +1,38 @@
import json
import httpretty
import waffle
from django.core.cache import cache
from django.core.management import call_command
from openedx.core.djangoapps.catalog.cache import PROGRAM_CACHE_KEY_TPL, PROGRAM_UUIDS_CACHE_KEY
from openedx.core.djangoapps.catalog.cache import (
PROGRAM_CACHE_KEY_TPL,
PROGRAM_UUIDS_CACHE_KEY,
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
)
from openedx.core.djangoapps.catalog.tests.factories import ProgramFactory
from openedx.core.djangoapps.catalog.tests.mixins import CatalogIntegrationMixin
from openedx.core.djangoapps.site_configuration.tests.mixins import SiteMixin
from openedx.core.djangolib.testing.utils import CacheIsolationTestCase, skip_unless_lms
from student.tests.factories import UserFactory
@skip_unless_lms
@httpretty.activate
class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase, SiteMixin):
ENABLED_CACHES = ['default']
def setUp(self):
super(TestCachePrograms, self).setUp()
self.catalog_integration = self.create_catalog_integration()
self.site_domain = 'testsite.com'
self.set_up_site(
self.site_domain,
{
'COURSE_CATALOG_API_URL': self.catalog_integration.get_internal_api_url().rstrip('/')
}
)
self.list_url = self.catalog_integration.get_internal_api_url().rstrip('/') + '/programs/'
self.detail_tpl = self.list_url.rstrip('/') + '/{uuid}/'
@@ -61,6 +74,7 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
content_type='application/json'
)
@waffle.testutils.override_switch('populate-multitenant-programs', True)
def test_handle(self):
"""
Verify that the command requests and caches program UUIDs and details.
@@ -83,7 +97,7 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
call_command('cache_programs')
cached_uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY)
cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
self.assertEqual(
set(cached_uuids),
set(self.uuids)
@@ -104,6 +118,7 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
for key, program in cached_programs.items():
self.assertEqual(program, programs[key])
@waffle.testutils.override_switch('populate-multitenant-programs', True)
def test_handle_missing_service_user(self):
"""
Verify that the command raises an exception when run without a service
@@ -112,9 +127,10 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
with self.assertRaises(Exception):
call_command('cache_programs')
cached_uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY)
cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
self.assertEqual(cached_uuids, None)
@waffle.testutils.override_switch('populate-multitenant-programs', True)
def test_handle_missing_uuids(self):
"""
Verify that the command raises an exception when it fails to retrieve
@@ -122,12 +138,14 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
"""
UserFactory(username=self.catalog_integration.service_username)
with self.assertRaises(Exception):
with self.assertRaises(SystemExit) as context:
call_command('cache_programs')
self.assertEqual(context.exception.code, 1)
cached_uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY)
self.assertEqual(cached_uuids, None)
cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
self.assertEqual(cached_uuids, [])
@waffle.testutils.override_switch('populate-multitenant-programs', True)
def test_handle_missing_programs(self):
"""
Verify that a problem retrieving a program doesn't prevent the command
@@ -154,7 +172,7 @@ class TestCachePrograms(CatalogIntegrationMixin, CacheIsolationTestCase):
self.assertEqual(context.exception.code, 1)
cached_uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY)
cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
self.assertEqual(
set(cached_uuids),
set(self.uuids)

37
openedx/core/djangoapps/catalog/utils.py
View File

@@ -2,26 +2,36 @@
import copy
import logging
import waffle
from django.conf import settings
from django.core.cache import cache
from django.core.exceptions import ObjectDoesNotExist
from edx_rest_api_client.client import EdxRestApiClient
from openedx.core.djangoapps.catalog.cache import PROGRAM_CACHE_KEY_TPL, PROGRAM_UUIDS_CACHE_KEY
from openedx.core.djangoapps.catalog.cache import (
PROGRAM_CACHE_KEY_TPL,
PROGRAM_UUIDS_CACHE_KEY,
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL
)
from openedx.core.djangoapps.catalog.models import CatalogIntegration
from openedx.core.djangoapps.theming.helpers import get_current_site
from openedx.core.lib.edx_api_utils import get_edx_api_data
from openedx.core.lib.token_utils import JwtBuilder
logger = logging.getLogger(__name__)
def create_catalog_api_client(user):
def create_catalog_api_client(user, site=None):
"""Returns an API client which can be used to make Catalog API requests."""
scopes = ['email', 'profile']
expires_in = settings.OAUTH_ID_TOKEN_EXPIRATION
jwt = JwtBuilder(user).build_token(scopes, expires_in)
url = CatalogIntegration.current().get_internal_api_url()
if site:
url = site.configuration.get_value('COURSE_CATALOG_API_URL')
else:
url = CatalogIntegration.current().get_internal_api_url()
return EdxRestApiClient(url, jwt=jwt)
@@ -45,8 +55,10 @@ def get_programs(uuid=None):
logger.warning(missing_details_msg_tpl.format(uuid=uuid))
return program
uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY, [])
if waffle.switch_is_active('get-multitenant-programs'):
uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=get_current_site().domain), [])
else:
uuids = cache.get(PROGRAM_UUIDS_CACHE_KEY, [])
if not uuids:
logger.warning('Failed to get program UUIDs from the cache.')
@@ -109,16 +121,14 @@ def get_program_types(name=None):
return []
def get_programs_with_type(types=None, include_hidden=True):
def get_programs_with_type(include_hidden=True):
"""
Return the list of programs. You can filter the types of programs returned using the optional
types parameter. If no filter is provided, all programs of all types will be returned. In addition,
you can specify whether to include hidden programs using the optional include_hidden parameter.
Return the list of programs. You can filter the types of programs returned by using the optional
include_hidden parameter. By default hidden programs will be included.
The program dict is updated with the fully serialized program type.
Keyword Arguments:
types (list): List of program type slugs to filter by.
include_hidden (bool): whether to include hidden programs
Return:
@@ -130,12 +140,7 @@ def get_programs_with_type(types=None, include_hidden=True):
if programs:
program_types = {program_type['name']: program_type for program_type in get_program_types()}
for program in programs:
# This limited type filtering is sufficient for current needs and
# helps us avoid additional complexity when caching program data.
# However, if the need for additional filtering of programs should
# arise, consider using the cache_programs management command to
# cache the filtered lists of UUIDs.
if types and program['type'] not in types:
if program['type'] not in program_types:
continue
if program['hidden'] and not include_hidden: