From ed25521f6171cc148624401239a7307a378446fc Mon Sep 17 00:00:00 2001 From: Zainab Amir Date: Thu, 12 Mar 2020 12:19:52 +0500 Subject: [PATCH] Enable testing pipeline for video uploads to s3 (#23375) Add integration settings to enable upload of videos from from studio. Settings enable user to connect to s3 bucket using mfa and assume role functionality. PROD-1214 --- .../contentstore/views/tests/test_videos.py | 53 ++++++++++++++++- cms/djangoapps/contentstore/views/videos.py | 59 +++++++++++++++++-- .../video_pipeline/config/waffle.py | 8 ++- 3 files changed, 110 insertions(+), 10 deletions(-) diff --git a/cms/djangoapps/contentstore/views/tests/test_videos.py b/cms/djangoapps/contentstore/views/tests/test_videos.py index d9f96b50ae..8585e64768 100644 --- a/cms/djangoapps/contentstore/views/tests/test_videos.py +++ b/cms/djangoapps/contentstore/views/tests/test_videos.py @@ -10,7 +10,6 @@ import re from contextlib import contextmanager from datetime import datetime from functools import wraps -from six import StringIO import dateutil.parser import ddt @@ -28,6 +27,7 @@ from edxval.api import ( get_video_info ) from mock import Mock, patch +from six import StringIO from waffle.testutils import override_flag from contentstore.models import VideoUploadConfig @@ -38,13 +38,18 @@ from contentstore.views.videos import ( KEY_EXPIRATION_IN_SECONDS, VIDEO_IMAGE_UPLOAD_ENABLED, WAFFLE_SWITCHES, + AssumeRole, StatusDisplayStrings, TranscriptProvider, _get_default_video_image_url, convert_video_status ) from openedx.core.djangoapps.profile_images.tests.helpers import make_image_file -from openedx.core.djangoapps.video_pipeline.config.waffle import DEPRECATE_YOUTUBE, waffle_flags +from openedx.core.djangoapps.video_pipeline.config.waffle import ( + DEPRECATE_YOUTUBE, + ENABLE_DEVSTACK_VIDEO_UPLOADS, + waffle_flags +) from openedx.core.djangoapps.waffle_utils.models import WaffleFlagCourseOverrideModel from openedx.core.djangoapps.waffle_utils.testutils import override_waffle_flag from xmodule.modulestore.tests.factories import CourseFactory @@ -452,6 +457,45 @@ class VideosHandlerTestCase(VideoUploadTestMixin, CourseTestCase): response = json.loads(response.content.decode('utf-8')) self.assertEqual(response['error'], u'The file name for %s must contain only ASCII characters.' % file_name) + @patch('boto.s3.key.Key') + @patch('boto.s3.connection.S3Connection') + @override_flag(waffle_flags()[ENABLE_DEVSTACK_VIDEO_UPLOADS].namespaced_flag_name, active=True) + def test_assume_role_connection(self, mock_conn, mock_key): + files = [{'file_name': 'first.mp4', 'content_type': 'video/mp4'}] + credentials = { + 'access_key': 'test_key', + 'secret_key': 'test_secret', + 'session_token': 'test_session_token' + } + + bucket = Mock() + mock_conn.return_value = Mock(get_bucket=Mock(return_value=bucket)) + mock_key_instances = [ + Mock( + generate_url=Mock( + return_value='http://example.com/url_{}'.format(file_info['file_name']) + ) + ) + for file_info in files + ] + mock_key.side_effect = mock_key_instances + [Mock()] + + with patch.object(AssumeRole, 'get_instance') as assume_role: + assume_role.return_value.credentials = credentials + + response = self.client.post( + self.url, + json.dumps({'files': files}), + content_type='application/json' + ) + + self.assertEqual(response.status_code, 200) + mock_conn.assert_called_once_with( + aws_access_key_id=credentials['access_key'], + aws_secret_access_key=credentials['secret_key'], + security_token=credentials['session_token'] + ) + @override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret') @patch('boto.s3.key.Key') @patch('boto.s3.connection.S3Connection') @@ -496,7 +540,10 @@ class VideosHandlerTestCase(VideoUploadTestMixin, CourseTestCase): self.assertEqual(response.status_code, 200) response_obj = json.loads(response.content.decode('utf-8')) - mock_conn.assert_called_once_with(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) + mock_conn.assert_called_once_with( + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY + ) self.assertEqual(len(response_obj['files']), len(files)) self.assertEqual(mock_key.call_count, len(files)) for i, file_info in enumerate(files): diff --git a/cms/djangoapps/contentstore/views/videos.py b/cms/djangoapps/contentstore/views/videos.py index 1782a3dab8..c0477cfee9 100644 --- a/cms/djangoapps/contentstore/views/videos.py +++ b/cms/djangoapps/contentstore/views/videos.py @@ -13,6 +13,7 @@ from uuid import uuid4 import rfc6266_parser import six from boto import s3 +from boto.sts import STSConnection from django.conf import settings from django.contrib.auth.decorators import login_required from django.contrib.staticfiles.storage import staticfiles_storage @@ -44,7 +45,11 @@ from contentstore.utils import reverse_course_url from contentstore.video_utils import validate_video_image from edxmako.shortcuts import render_to_response from openedx.core.djangoapps.video_config.models import VideoTranscriptEnabledFlag -from openedx.core.djangoapps.video_pipeline.config.waffle import DEPRECATE_YOUTUBE, waffle_flags +from openedx.core.djangoapps.video_pipeline.config.waffle import ( + DEPRECATE_YOUTUBE, + ENABLE_DEVSTACK_VIDEO_UPLOADS, + waffle_flags +) from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag, WaffleFlagNamespace, WaffleSwitchNamespace from util.json_request import JsonResponse, expect_json from xmodule.video_module.transcripts_utils import Transcript @@ -91,6 +96,38 @@ MAX_UPLOAD_HOURS = 24 VIDEOS_PER_PAGE = 100 +class AssumeRole(object): + """ Singleton class to establish connection to aws using mfa and assume role """ + __instance = None + + @staticmethod + def get_instance(): + """ Static access method. """ + if not AssumeRole.__instance: + AssumeRole() + + return AssumeRole.__instance + + def __init__(self): + """ Virtually private constructor. """ + if AssumeRole.__instance: + raise Exception("This is a singleton class!") + + sts = STSConnection( + settings.AWS_ACCESS_KEY_ID, + settings.AWS_SECRET_ACCESS_KEY + ) + self.credentials = sts.assume_role( + role_arn=settings.ROLE_ARN, + role_session_name='vem', + duration_seconds=3600, + mfa_serial_number=settings.MFA_SERIAL_NUMBER, + mfa_token=settings.MFA_TOKEN + ).credentials.to_dict() + + AssumeRole.__instance = self + + class TranscriptProvider(object): """ Transcription Provider Enumeration @@ -271,7 +308,7 @@ def validate_transcript_preferences(provider, cielo24_fidelity, cielo24_turnarou return error, preferences if not preferred_languages or not set(preferred_languages) <= set(supported_languages.keys()): - error = 'Invalid languages {}.'.format(preferred_languages) # pylint: disable=unicode-format-string + error = 'Invalid languages {}.'.format(preferred_languages) return error, preferences # Validated Cielo24 preferences @@ -765,10 +802,20 @@ def storage_service_bucket(): """ Returns an S3 bucket for video uploads. """ - conn = s3.connection.S3Connection( - settings.AWS_ACCESS_KEY_ID, - settings.AWS_SECRET_ACCESS_KEY - ) + if waffle_flags()[ENABLE_DEVSTACK_VIDEO_UPLOADS].is_enabled(): + credentials = AssumeRole.get_instance().credentials + params = { + 'aws_access_key_id': credentials['access_key'], + 'aws_secret_access_key': credentials['secret_key'], + 'security_token': credentials['session_token'] + } + else: + params = { + 'aws_access_key_id': settings.AWS_ACCESS_KEY_ID, + 'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY + } + + conn = s3.connection.S3Connection(**params) # We don't need to validate our bucket, it requires a very permissive IAM permission # set since behind the scenes it fires a HEAD request that is equivalent to get_all_keys() # meaning it would need ListObjects on the whole bucket, not just the path used in each diff --git a/openedx/core/djangoapps/video_pipeline/config/waffle.py b/openedx/core/djangoapps/video_pipeline/config/waffle.py index 022ce38de9..7c8e47e6cb 100644 --- a/openedx/core/djangoapps/video_pipeline/config/waffle.py +++ b/openedx/core/djangoapps/video_pipeline/config/waffle.py @@ -3,13 +3,14 @@ This module contains configuration settings via waffle flags for the Video Pipeline app. """ -from openedx.core.djangoapps.waffle_utils import WaffleFlagNamespace, CourseWaffleFlag +from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag, WaffleFlag, WaffleFlagNamespace # Videos Namespace WAFFLE_NAMESPACE = 'videos' # Waffle flag telling whether youtube is deprecated. DEPRECATE_YOUTUBE = 'deprecate_youtube' +ENABLE_DEVSTACK_VIDEO_UPLOADS = 'enable_devstack_video_uploads' def waffle_flags(): @@ -21,5 +22,10 @@ def waffle_flags(): DEPRECATE_YOUTUBE: CourseWaffleFlag( waffle_namespace=namespace, flag_name=DEPRECATE_YOUTUBE + ), + ENABLE_DEVSTACK_VIDEO_UPLOADS: WaffleFlag( + waffle_namespace=namespace, + flag_name=ENABLE_DEVSTACK_VIDEO_UPLOADS, + flag_undefined_default=False ) }