Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Save credentials in edx-video-pipeline

Browse Source 
EDUCATOR-1335 - This Adds a dedicated app which is responsible for communication with edx-video-pipeline service. It also adds the backend for saving 3rd party transcription service credentials on edx-video-pipline and cache it in edx-val.
This commit is contained in:
Qubad786
2017-10-04 20:12:53 +05:00
committed by Mushtaq Ali
parent f3b9ea4311
commit 2310614bb6
17 changed files with 564 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
cms/djangoapps/contentstore/views/__init__.py
View File

@@ -19,6 +19,7 @@ from .export_git import *
from .user import *
from .tabs import *
from .videos import *
from .transcript_settings import *
from .transcripts_ajax import *
try:
from .dev import *

171
cms/djangoapps/contentstore/views/tests/test_transcript_settings.py Normal file
View File

@@ -0,0 +1,171 @@
import ddt
import json
from mock import Mock, patch
from django.test.testcases import TestCase
from contentstore.tests.utils import CourseTestCase
from contentstore.utils import reverse_course_url
from contentstore.views.transcript_settings import TranscriptionProviderErrorType, validate_transcript_credentials
@ddt.ddt
@patch(
'openedx.core.djangoapps.video_config.models.VideoTranscriptEnabledFlag.feature_enabled',
Mock(return_value=True)
)
class TranscriptCredentialsTest(CourseTestCase):
"""
Tests for transcript credentials handler.
"""
VIEW_NAME = 'transcript_credentials_handler'
def get_url_for_course_key(self, course_id):
return reverse_course_url(self.VIEW_NAME, course_id)
def test_302_with_anonymous_user(self):
"""
Verify that redirection happens in case of unauthorized request.
"""
self.client.logout()
transcript_credentials_url = self.get_url_for_course_key(self.course.id)
response = self.client.post(transcript_credentials_url, content_type='application/json')
self.assertEqual(response.status_code, 302)
def test_405_with_not_allowed_request_method(self):
"""
Verify that 405 is returned in case of not-allowed request methods.
Allowed request methods include POST.
"""
transcript_credentials_url = self.get_url_for_course_key(self.course.id)
response = self.client.get(transcript_credentials_url, content_type='application/json')
self.assertEqual(response.status_code, 405)
def test_404_with_feature_disabled(self):
"""
Verify that 404 is returned if the corresponding feature is disabled.
"""
transcript_credentials_url = self.get_url_for_course_key(self.course.id)
with patch('openedx.core.djangoapps.video_config.models.VideoTranscriptEnabledFlag.feature_enabled') as feature:
feature.return_value = False
response = self.client.post(transcript_credentials_url, content_type='application/json')
self.assertEqual(response.status_code, 404)
@ddt.data(
(
{
'provider': 'abc_provider',
'api_key': '1234'
},
({}, None),
400,
'{\n "error": "Invalid Provider abc_provider."\n}'
),
(
{
'provider': '3PlayMedia',
'api_key': '11111',
'api_secret_key': '44444'
},
({'error_type': TranscriptionProviderErrorType.INVALID_CREDENTIALS}, False),
400,
'{\n "error": "Transcript credentials are not valid."\n}'
),
(
{
'provider': 'Cielo24',
'api_key': '12345',
'username': 'test_user'
},
({}, True),
200,
''
)
)
@ddt.unpack
@patch('contentstore.views.transcript_settings.update_3rd_party_transcription_service_credentials')
def test_transcript_credentials_handler(self, request_payload, update_credentials_response, expected_status_code,
expected_response, mock_update_credentials):
"""
Tests that transcript credentials handler works as expected.
"""
mock_update_credentials.return_value = update_credentials_response
transcript_credentials_url = self.get_url_for_course_key(self.course.id)
response = self.client.post(
transcript_credentials_url,
data=json.dumps(request_payload),
content_type='application/json'
)
self.assertEqual(response.status_code, expected_status_code)
self.assertEqual(response.content, expected_response)
@ddt.ddt
class TranscriptCredentialsValidationTest(TestCase):
"""
Tests for credentials validations.
"""
@ddt.data(
(
'ABC',
{
'username': 'test_user',
'password': 'test_pass'
},
'Invalid Provider ABC.',
{}
),
(
'Cielo24',
{
'username': 'test_user'
},
'api_key must be specified.',
{}
),
(
'Cielo24',
{
'username': 'test_user',
'api_key': 'test_api_key',
'extra_param': 'extra_value'
},
'',
{
'username': 'test_user',
'api_key': 'test_api_key'
}
),
(
'3PlayMedia',
{
'username': 'test_user'
},
'api_key and api_secret_key must be specified.',
{}
),
(
'3PlayMedia',
{
'api_key': 'test_key',
'api_secret_key': 'test_secret',
'extra_param': 'extra_value'
},
'',
{
'api_key': 'test_key',
'api_secret_key': 'test_secret'
}
),
)
@ddt.unpack
def test_invalid_credentials(self, provider, credentials, expected_error_message, expected_validated_credentials):
"""
Test validation with invalid transcript credentials.
"""
error_message, validated_credentials = validate_transcript_credentials(provider, **credentials)
# Assert the results.
self.assertEqual(error_message, expected_error_message)
self.assertDictEqual(validated_credentials, expected_validated_credentials)

110
cms/djangoapps/contentstore/views/transcript_settings.py Normal file
View File

@@ -0,0 +1,110 @@
"""
Views related to the transcript preferences feature
"""
from django.contrib.auth.decorators import login_required
from django.http import HttpResponseNotFound
from django.utils.translation import ugettext as _
from django.views.decorators.http import require_POST
from edxval.api import (
get_3rd_party_transcription_plans,
update_transcript_credentials_state_for_org,
)
from opaque_keys.edx.keys import CourseKey
from openedx.core.djangoapps.video_config.models import VideoTranscriptEnabledFlag
from openedx.core.djangoapps.video_pipeline.api import update_3rd_party_transcription_service_credentials
from util.json_request import JsonResponse, expect_json
from contentstore.views.videos import TranscriptProvider
__all__ = ['transcript_credentials_handler']
class TranscriptionProviderErrorType:
"""
Transcription provider's error types enumeration.
"""
INVALID_CREDENTIALS = 1
def validate_transcript_credentials(provider, **credentials):
"""
Validates transcript credentials.
Validations:
Providers must be either 3PlayMedia or Cielo24.
In case of:
3PlayMedia - 'api_key' and 'api_secret_key' are required.
Cielo24 - 'api_key' and 'username' are required.
It ignores any extra/unrelated parameters passed in credentials and
only returns the validated ones.
"""
error_message, validated_credentials = '', {}
valid_providers = get_3rd_party_transcription_plans().keys()
if provider in valid_providers:
must_have_props = []
if provider == TranscriptProvider.THREE_PLAY_MEDIA:
must_have_props = ['api_key', 'api_secret_key']
elif provider == TranscriptProvider.CIELO24:
must_have_props = ['api_key', 'username']
missing = [must_have_prop for must_have_prop in must_have_props if must_have_prop not in credentials.keys()]
if missing:
error_message = u'{missing} must be specified.'.format(missing=' and '.join(missing))
return error_message, validated_credentials
validated_credentials.update({
prop: credentials[prop] for prop in must_have_props
})
else:
error_message = u'Invalid Provider {provider}.'.format(provider=provider)
return error_message, validated_credentials
@expect_json
@login_required
@require_POST
def transcript_credentials_handler(request, course_key_string):
"""
JSON view handler to update the transcript organization credentials.
Arguments:
request: WSGI request object
course_key_string: A course identifier to extract the org.
Returns:
- A 200 response if credentials are valid and successfully updated in edx-video-pipeline.
- A 404 response if transcript feature is not enabled for this course.
- A 400 if credentials do not pass validations, hence not updated in edx-video-pipeline.
"""
course_key = CourseKey.from_string(course_key_string)
if not VideoTranscriptEnabledFlag.feature_enabled(course_key):
return HttpResponseNotFound()
provider = request.json.pop('provider')
error_message, validated_credentials = validate_transcript_credentials(provider=provider, **request.json)
if error_message:
response = JsonResponse({'error': error_message}, status=400)
else:
# Send the validated credentials to edx-video-pipeline.
credentials_payload = dict(validated_credentials, org=course_key.org, provider=provider)
error_response, is_updated = update_3rd_party_transcription_service_credentials(**credentials_payload)
# Send appropriate response based on whether credentials were updated or not.
if is_updated:
# Cache credentials state in edx-val.
update_transcript_credentials_state_for_org(org=course_key.org, provider=provider, exists=is_updated)
response = JsonResponse(status=200)
else:
# Error response would contain error types and the following
# error type is received from edx-video-pipeline whenever we've
# got invalid credentials for a provider. Its kept this way because
# edx-video-pipeline doesn't support i18n translations yet.
error_type = error_response.get('error_type')
if error_type == TranscriptionProviderErrorType.INVALID_CREDENTIALS:
error_message = _('Transcript credentials are not valid.')
response = JsonResponse({'error': error_message}, status=400)
return response

54
cms/djangoapps/contentstore/views/videos.py
View File

@@ -1,11 +1,10 @@
"""
Views related to the video upload feature
"""
from contextlib import closing
import csv
import json
import logging
from contextlib import closing
from datetime import datetime, timedelta
from uuid import uuid4
@@ -18,40 +17,37 @@ from django.core.files.images import get_image_dimensions
from django.http import HttpResponse, HttpResponseNotFound
from django.utils.translation import ugettext as _
from django.utils.translation import ugettext_noop
from django.views.decorators.http import require_GET, require_POST, require_http_methods
from django.views.decorators.http import require_GET, require_http_methods, require_POST
from edxval.api import (
SortDirection,
VideoSortField,
create_video,
get_videos_for_course,
remove_video_for_course,
update_video_status,
update_video_image,
get_3rd_party_transcription_plans,
get_transcript_preferences,
create_or_update_transcript_preferences,
remove_transcript_preferences,
create_video,
get_3rd_party_transcription_plans,
get_transcript_credentials_state_for_org,
update_transcript_credentials_state_for_org,
get_transcript_preferences,
get_videos_for_course,
remove_transcript_preferences,
remove_video_for_course,
update_video_image,
update_video_status
)
from opaque_keys.edx.keys import CourseKey
from openedx.core.djangoapps.video_config.models import VideoTranscriptEnabledFlag
from openedx.core.djangoapps.waffle_utils import WaffleSwitchNamespace
from contentstore.models import VideoUploadConfig
from contentstore.utils import reverse_course_url
from edxmako.shortcuts import render_to_response
from openedx.core.djangoapps.video_config.models import VideoTranscriptEnabledFlag
from openedx.core.djangoapps.waffle_utils import WaffleSwitchNamespace
from util.json_request import JsonResponse, expect_json
from .course import get_course_and_check_access
__all__ = [
'videos_handler',
'video_encodings_download',
'video_images_handler',
'transcript_preferences_handler',
'transcript_credentials_handler'
]
LOGGER = logging.getLogger(__name__)
@@ -388,32 +384,6 @@ def transcript_preferences_handler(request, course_key_string):
return JsonResponse()
@expect_json
@login_required
@require_POST
def transcript_credentials_handler(request, course_key_string):
"""
JSON view handler to post the transcript organization credentials.
Arguments:
request: WSGI request object
course_key_string: string for course key
Returns: An empty success response or 404 if transcript feature is not enabled
"""
course_key = CourseKey.from_string(course_key_string)
if not VideoTranscriptEnabledFlag.feature_enabled(course_key):
return HttpResponseNotFound()
org = course_key.org
provider = request.json.get('provider')
# TODO: Send organization credentials to edx-pipeline end point.
credentials = update_transcript_credentials_state_for_org(org, provider, exists=True)
return JsonResponse()
@login_required
@require_GET
def video_encodings_download(request, course_key_string):

3
cms/envs/common.py
View File

@@ -947,6 +947,9 @@ INSTALLED_APPS = [
# Video module configs (This will be moved to Video once it becomes an XBlock)
'openedx.core.djangoapps.video_config',
# edX Video Pipeline integration
'openedx.core.djangoapps.video_pipeline',
# For CMS
'contentstore.apps.ContentstoreConfig',

3
lms/envs/common.py
View File

@@ -2078,6 +2078,9 @@ INSTALLED_APPS = [
# Video module configs (This will be moved to Video once it becomes an XBlock)
'openedx.core.djangoapps.video_config',
# edX Video Pipeline integration
'openedx.core.djangoapps.video_pipeline',
# Bookmarks
'openedx.core.djangoapps.bookmarks.apps.BookmarksConfig',

0
openedx/core/djangoapps/video_pipeline/__init__.py Normal file
View File

9
openedx/core/djangoapps/video_pipeline/admin.py Normal file
View File

@@ -0,0 +1,9 @@
"""
Django admin for Video Pipeline models.
"""
from config_models.admin import ConfigurationModelAdmin
from django.contrib import admin
from openedx.core.djangoapps.video_pipeline.models import VideoPipelineIntegration
admin.site.register(VideoPipelineIntegration, ConfigurationModelAdmin)

51
openedx/core/djangoapps/video_pipeline/api.py Normal file
View File

@@ -0,0 +1,51 @@
"""
API utils in order to communicate to edx-video-pipeline.
"""
import json
import logging
from django.core.exceptions import ObjectDoesNotExist
from slumber.exceptions import HttpClientError
from openedx.core.djangoapps.video_pipeline.models import VideoPipelineIntegration
from openedx.core.djangoapps.video_pipeline.utils import create_video_pipeline_api_client
log = logging.getLogger(__name__)
def update_3rd_party_transcription_service_credentials(**credentials_payload):
"""
Updates the 3rd party transcription service's credentials.
Arguments:
credentials_payload(dict): A payload containing org, provider and its credentials.
Returns:
A Boolean specifying whether the credentials were updated or not
and an error response received from pipeline.
"""
error_response, is_updated = {}, False
pipeline_integration = VideoPipelineIntegration.current()
if pipeline_integration.enabled:
try:
video_pipeline_user = pipeline_integration.get_service_user()
except ObjectDoesNotExist:
return error_response, is_updated
client = create_video_pipeline_api_client(user=video_pipeline_user, api_url=pipeline_integration.api_url)
try:
client.transcript_credentials.post(credentials_payload)
is_updated = True
except HttpClientError as ex:
is_updated = False
log.exception(
('[video-pipeline-service] Unable to update transcript credentials '
'-- org=%s -- provider=%s -- response=%s.'),
credentials_payload.get('org'),
credentials_payload.get('provider'),
ex.content,
)
error_response = json.loads(ex.content)
return error_response, is_updated

31
openedx/core/djangoapps/video_pipeline/migrations/0001_initial.py Normal file
View File

@@ -0,0 +1,31 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
from django.conf import settings
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='VideoPipelineIntegration',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('change_date', models.DateTimeField(auto_now_add=True, verbose_name='Change date')),
('enabled', models.BooleanField(default=False, verbose_name='Enabled')),
('api_url', models.URLField(help_text='edx-video-pipeline API URL.', verbose_name='Internal API URL')),
('service_username', models.CharField(default=b'video_pipeline_service_user', help_text='Username created for Video Pipeline Integration, e.g. video_pipeline_service_user.', max_length=100)),
('changed_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, editable=False, to=settings.AUTH_USER_MODEL, null=True, verbose_name='Changed by')),
],
options={
'ordering': ('-change_date',),
'abstract': False,
},
),
]

0
openedx/core/djangoapps/video_pipeline/migrations/__init__.py Normal file
View File

31
openedx/core/djangoapps/video_pipeline/models.py Normal file
View File

@@ -0,0 +1,31 @@
"""
Model to hold edx-video-pipeline configurations.
"""
from config_models.models import ConfigurationModel
from django.contrib.auth import get_user_model
from django.db import models
from django.utils.translation import ugettext_lazy as _
class VideoPipelineIntegration(ConfigurationModel):
"""
Manages configuration for connecting to the edx-video-pipeline service and using its API.
"""
api_url = models.URLField(
verbose_name=_('Internal API URL'),
help_text=_('edx-video-pipeline API URL.')
)
service_username = models.CharField(
max_length=100,
default='video_pipeline_service_user',
null=False,
blank=False,
help_text=_('Username created for Video Pipeline Integration, e.g. video_pipeline_service_user.')
)
def get_service_user(self):
# NOTE: We load the user model here to avoid issues at startup time that result from the hacks
# in lms/startup.py.
User = get_user_model() # pylint: disable=invalid-name
return User.objects.get(username=self.service_username)

0
openedx/core/djangoapps/video_pipeline/tests/__init__.py Normal file
View File

23
openedx/core/djangoapps/video_pipeline/tests/mixins.py Normal file
View File

@@ -0,0 +1,23 @@
"""
Mixins to test video pipeline integration.
"""
from openedx.core.djangoapps.video_pipeline.models import VideoPipelineIntegration
class VideoPipelineIntegrationMixin(object):
"""
Utility for working with the video pipeline service during testing.
"""
video_pipeline_integration_defaults = {
'enabled': True,
'api_url': 'https://video-pipeline.example.com/api/v1/',
'service_username': 'cms_video_pipeline_service_user',
}
def create_video_pipeline_integration(self, **kwargs):
"""
Creates a new `VideoPipelineIntegration` record with `video_pipeline_integration_defaults`,
and it can be updated with any provided overrides.
"""
fields = dict(self.video_pipeline_integration_defaults, **kwargs)
return VideoPipelineIntegration.objects.create(**fields)

99
openedx/core/djangoapps/video_pipeline/tests/test_api.py Normal file
View File

@@ -0,0 +1,99 @@
"""
Tests for Video Pipeline api utils.
"""
import ddt
import json
from mock import Mock, patch
from django.test.testcases import TestCase
from slumber.exceptions import HttpClientError
from student.tests.factories import UserFactory
from openedx.core.djangoapps.video_pipeline.api import update_3rd_party_transcription_service_credentials
from openedx.core.djangoapps.video_pipeline.tests.mixins import VideoPipelineIntegrationMixin
@ddt.ddt
class TestAPIUtils(VideoPipelineIntegrationMixin, TestCase):
"""
Tests for API Utils.
"""
def setUp(self):
self.pipeline_integration = self.create_video_pipeline_integration()
self.user = UserFactory(username=self.pipeline_integration.service_username)
def test_update_transcription_service_credentials_with_integration_disabled(self):
"""
Test updating the credentials when service integration is disabled.
"""
self.pipeline_integration.enabled = False
self.pipeline_integration.save()
__, is_updated = update_3rd_party_transcription_service_credentials()
self.assertFalse(is_updated)
def test_update_transcription_service_credentials_with_unknown_user(self):
"""
Test updating the credentials when expected service user is not registered.
"""
self.pipeline_integration.service_username = 'non_existent_user'
self.pipeline_integration.save()
__, is_updated = update_3rd_party_transcription_service_credentials()
self.assertFalse(is_updated)
@ddt.data(
{
'username': 'Jason_cielo_24',
'api_key': '12345678',
},
{
'api_key': '12345678',
'api_secret': '11111111',
}
)
@patch('openedx.core.djangoapps.video_pipeline.api.log')
@patch('openedx.core.djangoapps.video_pipeline.utils.EdxRestApiClient')
def test_update_transcription_service_credentials(self, credentials_payload, mock_client, mock_logger):
"""
Tests that the update transcription service credentials api util works as expected.
"""
# Mock the post request
mock_credentials_endpoint = mock_client.return_value.transcript_credentials
# Try updating the transcription service credentials
error_response, is_updated = update_3rd_party_transcription_service_credentials(**credentials_payload)
mock_credentials_endpoint.post.assert_called_with(credentials_payload)
# Making sure log.exception is not called.
self.assertDictEqual(error_response, {})
self.assertFalse(mock_logger.exception.called)
self.assertTrue(is_updated)
@patch('openedx.core.djangoapps.video_pipeline.api.log')
@patch('openedx.core.djangoapps.video_pipeline.utils.EdxRestApiClient')
def test_update_transcription_service_credentials_exceptions(self, mock_client, mock_logger):
"""
Tests that the update transcription service credentials logs the exception occurring
during communication with edx-video-pipeline.
"""
error_content = '{"error_type": "1"}'
# Mock the post request
mock_credentials_endpoint = mock_client.return_value.transcript_credentials
mock_credentials_endpoint.post = Mock(side_effect=HttpClientError(content=error_content))
# try updating the transcription service credentials
credentials_payload = {
'org': 'mit',
'provider': 'ABC Provider',
'api_key': '61c56a8d0'
}
error_response, is_updated = update_3rd_party_transcription_service_credentials(**credentials_payload)
mock_credentials_endpoint.post.assert_called_with(credentials_payload)
# Assert the results.
self.assertFalse(is_updated)
self.assertDictEqual(error_response, json.loads(error_content))
mock_logger.exception.assert_called_with(
'[video-pipeline-service] Unable to update transcript credentials -- org=%s -- provider=%s -- response=%s.',
credentials_payload['org'],
credentials_payload['provider'],
error_content
)

19
openedx/core/djangoapps/video_pipeline/utils.py Normal file
View File

@@ -0,0 +1,19 @@
from django.conf import settings
from edx_rest_api_client.client import EdxRestApiClient
from openedx.core.lib.token_utils import JwtBuilder
def create_video_pipeline_api_client(user, api_url):
"""
Returns an API client which can be used to make Video Pipeline API requests.
Arguments:
user(User): A requesting user.
api_url(unicode): It is video pipeline's API URL.
"""
jwt_token = JwtBuilder(user).build_token(
scopes=[],
expires_in=settings.OAUTH_ID_TOKEN_EXPIRATION
)
return EdxRestApiClient(api_url, jwt=jwt_token)

2
requirements/edx/base.txt
View File

@@ -54,7 +54,7 @@ edx-organizations==0.4.7
edx-rest-api-client==1.7.1
edx-search==1.1.0
edx-submissions==2.0.12
edxval==0.1.2
edxval==0.1.3
event-tracking==0.2.4
feedparser==5.1.3
firebase-token-generator==1.3.2