Added CCXCon app and CCXCon URL in course advanced settings
This commit is contained in:
Peter Pinch
2016-01-20 14:28:48 -05:00
21 changed files with 635 additions and 3 deletions

View File

@@ -92,6 +92,7 @@ class CourseMetadata(object):
# Do not show enable_ccx if feature is not enabled.
if not settings.FEATURES.get('CUSTOM_COURSES_EDX'):
filtered_list.append('enable_ccx')
filtered_list.append('ccx_connector')
return filtered_list

View File

@@ -379,3 +379,7 @@ MICROSITE_DATABASE_TEMPLATE_CACHE_TTL = ENV_TOKENS.get(
# OpenID Connect issuer ID. Normally the URL of the authentication endpoint.
OAUTH_OIDC_ISSUER = ENV_TOKENS['OAUTH_OIDC_ISSUER']
######################## CUSTOM COURSES for EDX CONNECTOR ######################
if FEATURES.get('CUSTOM_COURSES_EDX'):
INSTALLED_APPS += ('openedx.core.djangoapps.ccxcon',)

View File

@@ -313,3 +313,7 @@ FEATURES['ENABLE_TEAMS'] = True
# Dummy secret key for dev/test
SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'
######### custom courses #########
INSTALLED_APPS += ('openedx.core.djangoapps.ccxcon',)
FEATURES['CUSTOM_COURSES_EDX'] = True

View File

@@ -247,3 +247,7 @@ BROKER_URL = "{0}://{1}:{2}@{3}/{4}".format(CELERY_BROKER_TRANSPORT,
CELERY_BROKER_PASSWORD,
CELERY_BROKER_HOSTNAME,
CELERY_BROKER_VHOST)
######################## CUSTOM COURSES for EDX CONNECTOR ######################
if FEATURES.get('CUSTOM_COURSES_EDX'):
INSTALLED_APPS += ('openedx.core.djangoapps.ccxcon',)

View File

@@ -395,6 +395,16 @@ class CourseFields(object):
default=False,
scope=Scope.settings
)
ccx_connector = String(
# Translators: Custom Courses for edX (CCX) is an edX feature for re-using course content.
display_name=_("CCX Connector URL"),
# Translators: Custom Courses for edX (CCX) is an edX feature for re-using course content.
help=_(
"URL for CCX Connector application for managing creation of CCXs. (optional)."
" Ignored unless 'Enable CCX' is set to 'true'."
),
scope=Scope.settings, default=""
)
allow_anonymous = Boolean(
display_name=_("Allow Anonymous Discussion Posts"),
help=_("Enter true or false. If true, students can create discussion posts that are anonymous to all users."),

View File

@@ -675,7 +675,7 @@ ECOMMERCE_API_TIMEOUT = ENV_TOKENS.get('ECOMMERCE_API_TIMEOUT', ECOMMERCE_API_TI
##### Custom Courses for EdX #####
if FEATURES.get('CUSTOM_COURSES_EDX'):
INSTALLED_APPS += ('lms.djangoapps.ccx',)
INSTALLED_APPS += ('lms.djangoapps.ccx', 'openedx.core.djangoapps.ccxcon')
FIELD_OVERRIDE_PROVIDERS += (
'lms.djangoapps.ccx.overrides.CustomCoursesForEdxOverrideProvider',
)

View File

@@ -535,7 +535,7 @@ FACEBOOK_APP_ID = "Test"
FACEBOOK_API_VERSION = "v2.2"
######### custom courses #########
INSTALLED_APPS += ('lms.djangoapps.ccx',)
INSTALLED_APPS += ('lms.djangoapps.ccx', 'openedx.core.djangoapps.ccxcon')
FEATURES['CUSTOM_COURSES_EDX'] = True
# Set dummy values for profile image settings.

View File

@@ -298,7 +298,7 @@ GRADES_DOWNLOAD_ROUTING_KEY = HIGH_MEM_QUEUE
##### Custom Courses for EdX #####
if FEATURES.get('CUSTOM_COURSES_EDX'):
INSTALLED_APPS += ('lms.djangoapps.ccx',)
INSTALLED_APPS += ('lms.djangoapps.ccx', 'openedx.core.djangoapps.ccxcon')
FIELD_OVERRIDE_PROVIDERS += (
'lms.djangoapps.ccx.overrides.CustomCoursesForEdxOverrideProvider',
)

View File

@@ -0,0 +1,9 @@
"""
The ccxcon app contains the models and the APIs to interact
with the `CCX Connector`, an application external to openedx
that is used to interact with the CCX and their master courses.
The ccxcon app needs to be placed in `openedx.core.djangoapps`
because it will be used both in CMS and LMS.
"""
import openedx.core.djangoapps.ccxcon.signals

View File

@@ -0,0 +1,9 @@
"""
Admin site bindings for ccxcon
"""
from django.contrib import admin
from .models import CCXCon
admin.site.register(CCXCon)

View File

@@ -0,0 +1,159 @@
"""
Module containing API functions for the CCXCon
"""
import logging
import urlparse
from django.core.validators import URLValidator
from django.core.exceptions import ValidationError
from django.http import Http404
from oauthlib.oauth2 import BackendApplicationClient
from requests_oauthlib import OAuth2Session
from rest_framework.status import (
HTTP_200_OK,
HTTP_201_CREATED,
)
from lms.djangoapps.courseware.courses import get_course_by_id
from lms.djangoapps.instructor.access import list_with_level
from openedx.core.djangoapps.models.course_details import CourseDetails
from student.models import anonymous_id_for_user
from .models import CCXCon
log = logging.getLogger(__name__)
CCXCON_COURSEXS_URL = '/api/v1/coursexs/'
CCXCON_TOKEN_URL = '/o/token/'
CCXCON_REQUEST_TIMEOUT = 30
class CCXConnServerError(Exception):
"""
Custom exception to be raised in case there is any
issue with the request to the server
"""
def is_valid_url(url):
"""
Helper function used to check if a string is a valid url.
Args:
url (str): the url string to be validated
Returns:
bool: whether the url is valid or not
"""
validate = URLValidator()
try:
validate(url)
return True
except ValidationError:
return False
def get_oauth_client(server_token_url, client_id, client_secret):
"""
Function that creates an oauth client and fetches a token.
It intentionally doesn't handle errors.
Args:
server_token_url (str): server URL where to get an authentication token
client_id (str): oauth client ID
client_secret (str): oauth client secret
Returns:
OAuth2Session: an instance of OAuth2Session with a token
"""
if not is_valid_url(server_token_url):
return
client = BackendApplicationClient(client_id=client_id)
oauth_ccxcon = OAuth2Session(client=client)
oauth_ccxcon.fetch_token(
token_url=server_token_url,
client_id=client_id,
client_secret=client_secret,
timeout=CCXCON_REQUEST_TIMEOUT
)
return oauth_ccxcon
def course_info_to_ccxcon(course_key):
"""
Function that gathers informations about the course and
makes a post request to a CCXCon with the data.
Args:
course_key (CourseLocator): the master course key
"""
try:
course = get_course_by_id(course_key)
except Http404:
log.error('Master Course with key "%s" not found', unicode(course_key))
return
if not course.enable_ccx:
log.debug('ccx not enabled for course key "%s"', unicode(course_key))
return
if not course.ccx_connector:
log.debug('ccx connector not defined for course key "%s"', unicode(course_key))
return
if not is_valid_url(course.ccx_connector):
log.error(
'ccx connector URL "%s" for course key "%s" is not a valid URL.',
course.ccx_connector, unicode(course_key)
)
return
# get the oauth credential for this URL
try:
ccxcon = CCXCon.objects.get(url=course.ccx_connector)
except CCXCon.DoesNotExist:
log.error('ccx connector Oauth credentials not configured for URL "%s".', course.ccx_connector)
return
# get an oauth client with a valid token
oauth_ccxcon = get_oauth_client(
server_token_url=urlparse.urljoin(course.ccx_connector, CCXCON_TOKEN_URL),
client_id=ccxcon.oauth_client_id,
client_secret=ccxcon.oauth_client_secret
)
# get the entire list of instructors
course_instructors = list_with_level(course, 'instructor')
# get anonymous ids for each of them
course_instructors_ids = [anonymous_id_for_user(user, course_key) for user in course_instructors]
# extract the course details
course_details = CourseDetails.fetch(course_key)
payload = {
'course_id': unicode(course_key),
'title': course.display_name,
'author_name': None,
'overview': course_details.overview,
'description': course_details.short_description,
'image_url': course_details.course_image_asset_path,
'instructors': course_instructors_ids
}
headers = {'content-type': 'application/json'}
# make the POST request
add_course_url = urlparse.urljoin(course.ccx_connector, CCXCON_COURSEXS_URL)
resp = oauth_ccxcon.post(
url=add_course_url,
json=payload,
headers=headers,
timeout=CCXCON_REQUEST_TIMEOUT
)
if resp.status_code >= 500:
raise CCXConnServerError('Server returned error Status: %s, Content: %s', resp.status_code, resp.content)
if resp.status_code >= 400:
log.error("Error creating course on ccxcon. Status: %s, Content: %s", resp.status_code, resp.content)
# this API performs a POST request both for POST and PATCH, but the POST returns 201 and the PATCH returns 200
elif resp.status_code != HTTP_200_OK and resp.status_code != HTTP_201_CREATED:
log.error('Server returned unexpected status code %s', resp.status_code)
else:
log.debug('Request successful. Status: %s, Content: %s', resp.status_code, resp.content)

View File

@@ -0,0 +1,27 @@
"""
Initial migration
"""
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
"""
Initial migration for CCXCon model
"""
dependencies = [
]
operations = [
migrations.CreateModel(
name='CCXCon',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('url', models.URLField(unique=True, db_index=True)),
('oauth_client_id', models.CharField(max_length=255)),
('oauth_client_secret', models.CharField(max_length=255)),
('title', models.CharField(max_length=255)),
],
),
]

View File

@@ -0,0 +1,31 @@
"""
Models for the ccxcon
"""
from django.db import models
class CCXCon(models.Model):
"""
The definition of the CCXCon model.
This will store the url and the oauth key to access the REST APIs
on the CCX Connector.
"""
url = models.URLField(unique=True, db_index=True)
oauth_client_id = models.CharField(max_length=255)
oauth_client_secret = models.CharField(max_length=255)
title = models.CharField(max_length=255)
class Meta(object):
app_label = 'ccxcon'
verbose_name = 'CCX Connector'
verbose_name_plural = 'CCX Connectors'
def __repr__(self):
return '<CCXCon {}>'.format(self.title)
def __str__(self):
return self.title
def __unicode__(self):
return unicode(self.__str__())

View File

@@ -0,0 +1,18 @@
"""
Signal handler for posting course updated to CCXCon
"""
from django.dispatch.dispatcher import receiver
from xmodule.modulestore.django import SignalHandler
@receiver(SignalHandler.course_published, dispatch_uid='ccxcon_course_publish_handler')
def _listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable=unused-argument
"""
Listener for course_plublish events.
This listener takes care of submitting a task to update CCXCon
"""
# update the course information on ccxcon using celery
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
from openedx.core.djangoapps.ccxcon import tasks
tasks.update_ccxcon.delay(unicode(course_key))

View File

@@ -0,0 +1,45 @@
"""
This file contains celery tasks for ccxcon
"""
from celery.task import task # pylint: disable=no-name-in-module, import-error
from celery.utils.log import get_task_logger # pylint: disable=no-name-in-module, import-error
from requests.exceptions import (
ConnectionError,
HTTPError,
RequestException,
TooManyRedirects
)
from opaque_keys.edx.keys import CourseKey
from openedx.core.djangoapps.ccxcon import api
log = get_task_logger(__name__)
@task()
def update_ccxcon(course_id, cur_retry=0):
"""
Pass through function to update course information on CCXCon.
Takes care of retries in case of some specific exceptions.
Args:
course_id (str): string representing a course key
cur_retry (int): integer representing the current task retry
"""
course_key = CourseKey.from_string(course_id)
try:
api.course_info_to_ccxcon(course_key)
log.info('Course update to CCXCon returned no errors. Course key: %s', course_id)
except (ConnectionError, HTTPError, RequestException, TooManyRedirects, api.CCXConnServerError) as exp:
log.error('Course update to CCXCon failed for course_id %s with error: %s', course_id, exp)
# in case the maximum amount of retries has not been reached,
# insert another task delayed exponentially up to 5 retries
if cur_retry < 5:
update_ccxcon.apply_async(
kwargs={'course_id': course_id, 'cur_retry': cur_retry + 1},
countdown=10 ** cur_retry # number of seconds the task should be delayed
)
log.info('Requeued celery task for course key %s ; retry # %s', course_id, cur_retry + 1)

View File

@@ -0,0 +1,17 @@
"""
Dummy factories for tests
"""
from factory.django import DjangoModelFactory
from openedx.core.djangoapps.ccxcon.models import CCXCon
class CcxConFactory(DjangoModelFactory):
"""
Model factory for the CCXCon model
"""
class Meta(object):
model = CCXCon
oauth_client_id = 'asdfjasdljfasdkjffsdfjksd98fsd8y24fdsiuhsfdsf'
oauth_client_secret = '19123084091238901912308409123890'
title = 'title for test ccxcon'

View File

@@ -0,0 +1,214 @@
"""
Unit tests for the API module
"""
import datetime
import mock
import pytz
import urlparse
from nose.plugins.attrib import attr
from opaque_keys.edx.keys import CourseKey
from student.tests.factories import AdminFactory
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import (
SharedModuleStoreTestCase,
TEST_DATA_SPLIT_MODULESTORE
)
from xmodule.modulestore.tests.factories import (
CourseFactory,
ItemFactory,
)
from openedx.core.djangoapps.ccxcon import api as ccxconapi
from .factories import CcxConFactory
def flatten(seq):
"""
For [[1, 2], [3, 4]] returns [1, 2, 3, 4]. Does not recurse.
"""
return [x for sub in seq for x in sub]
def fetch_token_mock(*args, **kwargs): # pylint: disable=unused-argument
"""
Mock function used to bypass the oauth fetch token
"""
return
@attr('shard_1')
class APIsTestCase(SharedModuleStoreTestCase):
"""
Unit tests for the API module functions
"""
MODULESTORE = TEST_DATA_SPLIT_MODULESTORE
@classmethod
def setUpClass(cls):
super(APIsTestCase, cls).setUpClass()
cls.course = course = CourseFactory.create()
cls.course_key = cls.course.location.course_key
# Create a course outline
start = datetime.datetime(
2010, 5, 12, 2, 42, tzinfo=pytz.UTC
)
due = datetime.datetime(
2010, 7, 7, 0, 0, tzinfo=pytz.UTC
)
cls.chapters = [
ItemFactory.create(start=start, parent=course) for _ in xrange(2)
]
cls.sequentials = flatten([
[
ItemFactory.create(parent=chapter) for _ in xrange(2)
] for chapter in cls.chapters
])
cls.verticals = flatten([
[
ItemFactory.create(
start=start, due=due, parent=sequential, graded=True, format='Homework', category=u'vertical'
) for _ in xrange(2)
] for sequential in cls.sequentials
])
# Trying to wrap the whole thing in a bulk operation fails because it
# doesn't find the parents. But we can at least wrap this part...
with cls.store.bulk_operations(course.id, emit_signals=False):
blocks = flatten([ # pylint: disable=unused-variable
[
ItemFactory.create(parent=vertical) for _ in xrange(2)
] for vertical in cls.verticals
])
def setUp(self):
"""
Set up tests
"""
super(APIsTestCase, self).setUp()
# Create instructor account
self.instructor = AdminFactory.create()
# create an instance of modulestore
self.mstore = modulestore()
# enable ccx
self.course.enable_ccx = True
# setup CCX connector
self.course.ccx_connector = 'https://url.to.cxx.connector.mit.edu'
# save the changes
self.mstore.update_item(self.course, self.instructor.id)
# create a configuration for the ccx connector: this must match the one in the course
self.ccxcon_conf = CcxConFactory(url=self.course.ccx_connector)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.fetch_token', fetch_token_mock)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.post')
def test_course_info_to_ccxcon_no_valid_course_key(self, mock_post):
"""
Test for an invalid course key
"""
missing_course_key = CourseKey.from_string('course-v1:FakeOrganization+CN999+CR-FALL99')
self.assertIsNone(ccxconapi.course_info_to_ccxcon(missing_course_key))
self.assertEqual(mock_post.call_count, 0)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.fetch_token', fetch_token_mock)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.post')
def test_course_info_to_ccxcon_no_ccx_enabled(self, mock_post):
"""
Test for a course without CCX enabled
"""
self.course.enable_ccx = False
self.mstore.update_item(self.course, self.instructor.id)
self.assertIsNone(ccxconapi.course_info_to_ccxcon(self.course_key))
self.assertEqual(mock_post.call_count, 0)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.fetch_token', fetch_token_mock)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.post')
def test_course_info_to_ccxcon_invalid_ccx_connector(self, mock_post):
"""
Test for a course with invalid CCX connector URL
"""
# no connector at all
self.course.ccx_connector = ""
self.mstore.update_item(self.course, self.instructor.id)
self.assertIsNone(ccxconapi.course_info_to_ccxcon(self.course_key))
self.assertEqual(mock_post.call_count, 0)
# invalid url
self.course.ccx_connector = "www.foo"
self.mstore.update_item(self.course, self.instructor.id)
self.assertIsNone(ccxconapi.course_info_to_ccxcon(self.course_key))
self.assertEqual(mock_post.call_count, 0)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.fetch_token', fetch_token_mock)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.post')
def test_course_info_to_ccxcon_no_config(self, mock_post):
"""
Test for course with ccx connector credentials not configured
"""
self.course.ccx_connector = "https://www.foo.com"
self.mstore.update_item(self.course, self.instructor.id)
self.assertIsNone(ccxconapi.course_info_to_ccxcon(self.course_key))
self.assertEqual(mock_post.call_count, 0)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.fetch_token', fetch_token_mock)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.post')
def test_course_info_to_ccxcon_ok(self, mock_post):
"""
Test for happy path
"""
mock_response = mock.Mock()
mock_response.status_code = 201
mock_post.return_value = mock_response
ccxconapi.course_info_to_ccxcon(self.course_key)
self.assertEqual(mock_post.call_count, 1)
k_args, k_kwargs = mock_post.call_args
# no args used for the call
self.assertEqual(k_args, tuple())
self.assertEqual(
k_kwargs.get('url'),
urlparse.urljoin(self.course.ccx_connector, ccxconapi.CCXCON_COURSEXS_URL)
)
# second call with different status code
mock_response.status_code = 200
mock_post.return_value = mock_response
ccxconapi.course_info_to_ccxcon(self.course_key)
self.assertEqual(mock_post.call_count, 2)
k_args, k_kwargs = mock_post.call_args
# no args used for the call
self.assertEqual(k_args, tuple())
self.assertEqual(
k_kwargs.get('url'),
urlparse.urljoin(self.course.ccx_connector, ccxconapi.CCXCON_COURSEXS_URL)
)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.fetch_token', fetch_token_mock)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.post')
def test_course_info_to_ccxcon_500_error(self, mock_post):
"""
Test for 500 error: a CCXConnServerError exception is raised
"""
mock_response = mock.Mock()
mock_response.status_code = 500
mock_post.return_value = mock_response
with self.assertRaises(ccxconapi.CCXConnServerError):
ccxconapi.course_info_to_ccxcon(self.course_key)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.fetch_token', fetch_token_mock)
@mock.patch('requests_oauthlib.oauth2_session.OAuth2Session.post')
def test_course_info_to_ccxcon_other_status_codes(self, mock_post):
"""
Test for status codes different from >= 500 and 201:
The called function doesn't raise any exception and simply returns None.
"""
mock_response = mock.Mock()
for status_code in (204, 300, 304, 400, 404):
mock_response.status_code = status_code
mock_post.return_value = mock_response
self.assertIsNone(ccxconapi.course_info_to_ccxcon(self.course_key))

View File

@@ -0,0 +1,34 @@
"""
Test for contentstore signals receiver
"""
import mock
from django.test import TestCase
from opaque_keys.edx.keys import CourseKey
from xmodule.modulestore.django import modulestore, SignalHandler
class CCXConSignalTestCase(TestCase):
"""
The only tests currently implemented are for verifying that
the call for the ccxcon update are performed correctly by the
course_published signal handler
"""
@mock.patch('openedx.core.djangoapps.ccxcon.tasks.update_ccxcon.delay')
def test_course_published_ccxcon_call(self, mock_upc):
"""
Tests the async call to the ccxcon task.
It bypasses all the other calls.
"""
mock_response = mock.MagicMock(return_value=None)
mock_upc.return_value = mock_response
course_id = u'course-v1:OrgFoo+CN199+CR-FALL01'
course_key = CourseKey.from_string(course_id)
signal_handler = SignalHandler(modulestore())
signal_handler.send('course_published', course_key=course_key)
mock_upc.assert_called_once_with(course_id)

View File

@@ -0,0 +1,46 @@
"""
Tests for the CCXCon celery tasks
"""
import mock
from django.test import TestCase
from opaque_keys.edx.keys import CourseKey
from openedx.core.djangoapps.ccxcon import api, tasks
class CCXConTaskTestCase(TestCase):
"""
Tests for CCXCon tasks.
"""
@mock.patch('openedx.core.djangoapps.ccxcon.api.course_info_to_ccxcon')
def test_update_ccxcon_task_ok(self, mock_citc):
"""
Test task with no problems
"""
mock_response = mock.Mock()
mock_citc.return_value = mock_response
course_id = u'course-v1:OrgFoo+CN199+CR-FALL01'
tasks.update_ccxcon.delay(course_id)
mock_citc.assert_called_once_with(CourseKey.from_string(course_id))
@mock.patch('openedx.core.djangoapps.ccxcon.api.course_info_to_ccxcon')
def test_update_ccxcon_task_retry(self, mock_citc):
"""
Test task with exception that triggers a retry
"""
mock_citc.side_effect = api.CCXConnServerError()
course_id = u'course-v1:OrgFoo+CN199+CR-FALL01'
tasks.update_ccxcon.delay(course_id)
self.assertEqual(mock_citc.call_count, 6)
course_key = CourseKey.from_string(course_id)
for call in mock_citc.call_args_list:
c_args, c_kwargs = call
self.assertEqual(c_kwargs, {})
self.assertEqual(len(c_args), 1)
self.assertEqual(c_args[0], course_key)