Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Mgmt. Command to Sync Marketing Urls from Catalog service

Browse Source 
Introduces a mgmt. command that retrieves course runs from catalog service in order to update marketing urls for the courses found in course metadata cache (i.e. CourseOverview). This also provides an updated utility to retrieve course sharing url.
This commit is contained in:
Qubad786
2017-02-13 18:38:45 +05:00
parent 01ed17426f
commit 7c1f6ff2c0
18 changed files with 482 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
openedx/core/djangoapps/catalog/management/__init__.py Normal file
View File

0
openedx/core/djangoapps/catalog/management/commands/__init__.py Normal file
View File

70
openedx/core/djangoapps/catalog/management/commands/sync_course_runs.py Normal file
View File

@@ -0,0 +1,70 @@
"""
Sync course runs from catalog service.
"""
import logging
from django.core.management.base import BaseCommand
from opaque_keys.edx.keys import CourseKey
from openedx.core.djangoapps.catalog.utils import get_course_runs
from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
log = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Purpose is to sync course runs data from catalog service to make it accessible in edx-platform.
It just happens to only be syncing marketing URLs from catalog course runs for now.
"""
help = 'Refresh marketing urls from catalog service.'
def update_course_overviews(self, course_runs):
"""
Refresh marketing urls for the given catalog course runs.
Arguments:
course_runs: A list containing catalog course runs.
"""
# metrics for observability
# number of catalog course runs retrieved.
catalog_course_runs_retrieved = len(course_runs)
# number of catalog course runs found in course overview.
course_runs_found_in_cache = 0
# number of course overview records actually get updated.
course_metadata_updated = 0
for course_run in course_runs:
marketing_url = course_run['marketing_url']
course_key = CourseKey.from_string(course_run['key'])
try:
course_overview = CourseOverview.objects.get(id=course_key)
course_runs_found_in_cache += 1
except CourseOverview.DoesNotExist:
log.info(
'[sync_course_runs] course overview record not found for course run: %s',
unicode(course_key),
)
continue
# Check whether course overview's marketing url is outdated - this saves a db hit.
if course_overview.marketing_url != marketing_url:
course_overview.marketing_url = marketing_url
course_overview.save()
course_metadata_updated += 1
return catalog_course_runs_retrieved, course_runs_found_in_cache, course_metadata_updated
def handle(self, *args, **options):
log.info('[sync_course_runs] Fetching course runs from catalog service.')
course_runs = get_course_runs()
course_runs_retrieved, course_runs_found, course_metadata_updated = self.update_course_overviews(course_runs)
log.info(
('[sync_course_runs] course runs retrieved: %d, course runs found in course overview: %d,'
' course runs not found in course overview: %d, course overviews metadata updated: %d,'),
course_runs_retrieved,
course_runs_found,
course_runs_retrieved - course_runs_found,
course_metadata_updated,
)

0
openedx/core/djangoapps/catalog/management/commands/tests/__init__.py Normal file
View File

89
openedx/core/djangoapps/catalog/management/commands/tests/test_sync_course_runs.py Normal file
View File

@@ -0,0 +1,89 @@
"""
Tests for the sync course runs management command.
"""
import ddt
import mock
from django.core.management import call_command
from openedx.core.djangoapps.catalog.tests.factories import CourseRunFactory
from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
COMMAND_MODULE = 'openedx.core.djangoapps.catalog.management.commands.sync_course_runs'
@ddt.ddt
@mock.patch(COMMAND_MODULE + '.get_course_runs')
class TestSyncCourseRunsCommand(ModuleStoreTestCase):
"""
Test for the sync course runs management command.
"""
def setUp(self):
super(TestSyncCourseRunsCommand, self).setUp()
# create mongo course
self.course = CourseFactory.create()
# load this course into course overview
CourseOverview.get_from_id(self.course.id)
# create a catalog course run with the same course id.
self.catalog_course_run = CourseRunFactory(
key=unicode(self.course.id),
marketing_url='test_marketing_url'
)
def get_course_overview_marketing_url(self, course_id):
"""
Get course overview marketing url.
"""
return CourseOverview.objects.get(id=course_id).marketing_url
def test_marketing_url_on_sync(self, mock_catalog_course_runs):
"""
Verify the updated marketing url on execution of the management command.
"""
mock_catalog_course_runs.return_value = [self.catalog_course_run]
earlier_marketing_url = self.get_course_overview_marketing_url(self.course.id)
call_command('sync_course_runs')
updated_marketing_url = self.get_course_overview_marketing_url(self.course.id)
# Assert that the Marketing URL has changed.
self.assertNotEqual(earlier_marketing_url, updated_marketing_url)
self.assertEqual(updated_marketing_url, 'test_marketing_url')
@mock.patch(COMMAND_MODULE + '.log.info')
def test_course_overview_does_not_exist(self, mock_log_info, mock_catalog_course_runs):
"""
Verify no error in case if a course run is not found in course overview.
"""
nonexistent_course_run = CourseRunFactory()
mock_catalog_course_runs.return_value = [self.catalog_course_run, nonexistent_course_run]
call_command('sync_course_runs')
mock_log_info.assert_any_call(
'[sync_course_runs] course overview record not found for course run: %s',
nonexistent_course_run['key'],
)
updated_marketing_url = self.get_course_overview_marketing_url(self.course.id)
self.assertEqual(updated_marketing_url, 'test_marketing_url')
@mock.patch(COMMAND_MODULE + '.log.info')
def test_starting_and_ending_logs(self, mock_log_info, mock_catalog_course_runs):
"""
Verify logging at start and end of the command.
"""
mock_catalog_course_runs.return_value = [self.catalog_course_run, CourseRunFactory(), CourseRunFactory()]
call_command('sync_course_runs')
# Assert the logs at the start of the command.
mock_log_info.assert_any_call('[sync_course_runs] Fetching course runs from catalog service.')
# Assert the log metrics at it's completion.
mock_log_info.assert_any_call(
('[sync_course_runs] course runs retrieved: %d, course runs found in course overview: %d,'
' course runs not found in course overview: %d, course overviews metadata updated: %d,'),
3,
1,
2,
1,
)

19
openedx/core/djangoapps/catalog/migrations/0003_catalogintegration_page_size.py Normal file
View File

@@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('catalog', '0002_catalogintegration_username'),
]
operations = [
migrations.AddField(
model_name='catalogintegration',
name='page_size',
field=models.PositiveIntegerField(default=100, help_text='Maximum number of records in paginated response of a single request to catalog service.', verbose_name='Page Size'),
),
]

8
openedx/core/djangoapps/catalog/models.py
View File

@@ -35,6 +35,14 @@ class CatalogIntegration(ConfigurationModel):
)
)
page_size = models.PositiveIntegerField(
verbose_name=_('Page Size'),
default=100,
help_text=_(
'Maximum number of records in paginated response of a single request to catalog service.'
)
)
@property
def is_cache_enabled(self):
"""Whether responses from the catalog API will be cached."""

1
openedx/core/djangoapps/catalog/tests/mixins.py
View File

@@ -10,6 +10,7 @@ class CatalogIntegrationMixin(object):
'internal_api_url': 'https://catalog-internal.example.com/api/v1/',
'cache_ttl': 0,
'service_username': 'lms_catalog_service_user',
'page_size': 20,
}
def create_catalog_integration(self, **kwargs):

73
openedx/core/djangoapps/catalog/tests/test_utils.py
View File

@@ -8,12 +8,13 @@ from django.test import TestCase
import mock
from openedx.core.djangoapps.catalog.models import CatalogIntegration
from openedx.core.djangoapps.catalog.tests.factories import ProgramFactory, ProgramTypeFactory
from openedx.core.djangoapps.catalog.tests.factories import CourseRunFactory, ProgramFactory, ProgramTypeFactory
from openedx.core.djangoapps.catalog.tests.mixins import CatalogIntegrationMixin
from openedx.core.djangoapps.catalog.utils import (
get_programs,
get_program_types,
get_programs_with_type,
get_course_runs,
)
from openedx.core.djangolib.testing.utils import skip_unless_lms
from student.tests.factories import UserFactory
@@ -176,3 +177,73 @@ class TestGetProgramTypes(CatalogIntegrationMixin, TestCase):
program = program_types[0]
data = get_program_types(name=program['name'])
self.assertEqual(data, program)
@skip_unless_lms
@mock.patch(UTILS_MODULE + '.get_edx_api_data')
class TestGetCourseRuns(CatalogIntegrationMixin, TestCase):
"""
Tests covering retrieval of course runs from the catalog service.
"""
def setUp(self):
super(TestGetCourseRuns, self).setUp()
self.catalog_integration = self.create_catalog_integration(cache_ttl=1)
self.user = UserFactory(username=self.catalog_integration.service_username)
def assert_contract(self, call_args): # pylint: disable=redefined-builtin
"""
Verify that API data retrieval utility is used correctly.
"""
args, kwargs = call_args
for arg in (self.catalog_integration, self.user, 'course_runs'):
self.assertIn(arg, args)
self.assertEqual(kwargs['api']._store['base_url'], self.catalog_integration.internal_api_url) # pylint: disable=protected-access
querystring = {
'page_size': 20,
'exclude_utm': 1,
}
self.assertEqual(kwargs['querystring'], querystring)
return args, kwargs
def test_config_missing(self, mock_get_edx_api_data):
"""
Verify that no errors occur when catalog config is missing.
"""
CatalogIntegration.objects.all().delete()
data = get_course_runs()
self.assertFalse(mock_get_edx_api_data.called)
self.assertEqual(data, [])
@mock.patch(UTILS_MODULE + '.log.error')
def test_service_user_missing(self, mock_log_error, mock_get_edx_api_data):
"""
Verify that no errors occur when the catalog service user is missing.
"""
catalog_integration = self.create_catalog_integration(service_username='nonexistent-user')
data = get_course_runs()
mock_log_error.any_call(
'Catalog service user with username [%s] does not exist. Course runs will not be retrieved.',
catalog_integration.service_username,
)
self.assertFalse(mock_get_edx_api_data.called)
self.assertEqual(data, [])
def test_get_course_runs(self, mock_get_edx_api_data):
"""
Test retrieval of course runs.
"""
catalog_course_runs = [CourseRunFactory() for __ in xrange(10)]
mock_get_edx_api_data.return_value = catalog_course_runs
data = get_course_runs()
self.assertTrue(mock_get_edx_api_data.called)
self.assert_contract(mock_get_edx_api_data.call_args)
self.assertEqual(data, catalog_course_runs)

40
openedx/core/djangoapps/catalog/utils.py
View File

@@ -1,5 +1,6 @@
"""Helper functions for working with the catalog service."""
import copy
import logging
from django.conf import settings
from django.contrib.auth import get_user_model
@@ -10,6 +11,8 @@ from openedx.core.lib.edx_api_utils import get_edx_api_data
from openedx.core.lib.token_utils import JwtBuilder
log = logging.getLogger(__name__)
User = get_user_model() # pylint: disable=invalid-name
@@ -135,3 +138,40 @@ def get_programs_with_type(types=None):
programs_with_type.append(program_with_type)
return programs_with_type
def get_course_runs():
"""
Retrieve all the course runs from the catalog service.
Returns:
list of dict with each record representing a course run.
"""
catalog_integration = CatalogIntegration.current()
course_runs = []
if catalog_integration.enabled:
try:
user = User.objects.get(username=catalog_integration.service_username)
except User.DoesNotExist:
log.error(
'Catalog service user with username [%s] does not exist. Course runs will not be retrieved.',
catalog_integration.service_username,
)
return course_runs
api = create_catalog_api_client(user, catalog_integration)
querystring = {
'page_size': catalog_integration.page_size,
'exclude_utm': 1,
}
course_runs = get_edx_api_data(
catalog_integration,
user,
'course_runs',
api=api,
querystring=querystring,
)
return course_runs

19
openedx/core/djangoapps/content/course_overviews/migrations/0011_courseoverview_marketing_url.py Normal file
View File

@@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('course_overviews', '0010_auto_20160329_2317'),
]
operations = [
migrations.AddField(
model_name='courseoverview',
name='marketing_url',
field=models.TextField(null=True),
),
]

1
openedx/core/djangoapps/content/course_overviews/models.py
View File

@@ -97,6 +97,7 @@ class CourseOverview(TimeStampedModel):
course_video_url = TextField(null=True)
effort = TextField(null=True)
self_paced = BooleanField(default=False)
marketing_url = TextField(null=True)
@classmethod
def _create_from_course(cls, course):