Enable testing pipeline for video uploads to s3 (#23375)

Add integration settings to enable upload of videos from from studio.
Settings enable user to connect to s3 bucket using mfa and assume role
functionality.

PROD-1214

cms/djangoapps/contentstore/views/tests/test_videos.py

@@ -10,7 +10,6 @@ import re
 from contextlib import contextmanager
 from datetime import datetime
 from functools import wraps
-from six import StringIO
 
 import dateutil.parser
 import ddt
@@ -28,6 +27,7 @@ from edxval.api import (
     get_video_info
 )
 from mock import Mock, patch
+from six import StringIO
 from waffle.testutils import override_flag
 
 from contentstore.models import VideoUploadConfig
@@ -38,13 +38,18 @@ from contentstore.views.videos import (
     KEY_EXPIRATION_IN_SECONDS,
     VIDEO_IMAGE_UPLOAD_ENABLED,
     WAFFLE_SWITCHES,
+    AssumeRole,
     StatusDisplayStrings,
     TranscriptProvider,
     _get_default_video_image_url,
     convert_video_status
 )
 from openedx.core.djangoapps.profile_images.tests.helpers import make_image_file
-from openedx.core.djangoapps.video_pipeline.config.waffle import DEPRECATE_YOUTUBE, waffle_flags
+from openedx.core.djangoapps.video_pipeline.config.waffle import (
+    DEPRECATE_YOUTUBE,
+    ENABLE_DEVSTACK_VIDEO_UPLOADS,
+    waffle_flags
+)
 from openedx.core.djangoapps.waffle_utils.models import WaffleFlagCourseOverrideModel
 from openedx.core.djangoapps.waffle_utils.testutils import override_waffle_flag
 from xmodule.modulestore.tests.factories import CourseFactory
@@ -452,6 +457,45 @@ class VideosHandlerTestCase(VideoUploadTestMixin, CourseTestCase):
         response = json.loads(response.content.decode('utf-8'))
         self.assertEqual(response['error'], u'The file name for %s must contain only ASCII characters.' % file_name)
 
+    @patch('boto.s3.key.Key')
+    @patch('boto.s3.connection.S3Connection')
+    @override_flag(waffle_flags()[ENABLE_DEVSTACK_VIDEO_UPLOADS].namespaced_flag_name, active=True)
+    def test_assume_role_connection(self, mock_conn, mock_key):
+        files = [{'file_name': 'first.mp4', 'content_type': 'video/mp4'}]
+        credentials = {
+            'access_key': 'test_key',
+            'secret_key': 'test_secret',
+            'session_token': 'test_session_token'
+        }
+
+        bucket = Mock()
+        mock_conn.return_value = Mock(get_bucket=Mock(return_value=bucket))
+        mock_key_instances = [
+            Mock(
+                generate_url=Mock(
+                    return_value='http://example.com/url_{}'.format(file_info['file_name'])
+                )
+            )
+            for file_info in files
+        ]
+        mock_key.side_effect = mock_key_instances + [Mock()]
+
+        with patch.object(AssumeRole, 'get_instance') as assume_role:
+            assume_role.return_value.credentials = credentials
+
+            response = self.client.post(
+                self.url,
+                json.dumps({'files': files}),
+                content_type='application/json'
+            )
+
+            self.assertEqual(response.status_code, 200)
+            mock_conn.assert_called_once_with(
+                aws_access_key_id=credentials['access_key'],
+                aws_secret_access_key=credentials['secret_key'],
+                security_token=credentials['session_token']
+            )
+
     @override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret')
     @patch('boto.s3.key.Key')
     @patch('boto.s3.connection.S3Connection')
@@ -496,7 +540,10 @@ class VideosHandlerTestCase(VideoUploadTestMixin, CourseTestCase):
         self.assertEqual(response.status_code, 200)
         response_obj = json.loads(response.content.decode('utf-8'))
 
-        mock_conn.assert_called_once_with(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
+        mock_conn.assert_called_once_with(
+            aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
+            aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY
+        )
         self.assertEqual(len(response_obj['files']), len(files))
         self.assertEqual(mock_key.call_count, len(files))
         for i, file_info in enumerate(files):

cms/djangoapps/contentstore/views/videos.py

@@ -13,6 +13,7 @@ from uuid import uuid4
 import rfc6266_parser
 import six
 from boto import s3
+from boto.sts import STSConnection
 from django.conf import settings
 from django.contrib.auth.decorators import login_required
 from django.contrib.staticfiles.storage import staticfiles_storage
@@ -44,7 +45,11 @@ from contentstore.utils import reverse_course_url
 from contentstore.video_utils import validate_video_image
 from edxmako.shortcuts import render_to_response
 from openedx.core.djangoapps.video_config.models import VideoTranscriptEnabledFlag
-from openedx.core.djangoapps.video_pipeline.config.waffle import DEPRECATE_YOUTUBE, waffle_flags
+from openedx.core.djangoapps.video_pipeline.config.waffle import (
+    DEPRECATE_YOUTUBE,
+    ENABLE_DEVSTACK_VIDEO_UPLOADS,
+    waffle_flags
+)
 from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag, WaffleFlagNamespace, WaffleSwitchNamespace
 from util.json_request import JsonResponse, expect_json
 from xmodule.video_module.transcripts_utils import Transcript
@@ -91,6 +96,38 @@ MAX_UPLOAD_HOURS = 24
 VIDEOS_PER_PAGE = 100
 
 
+class AssumeRole(object):
+    """ Singleton class to establish connection to aws using mfa and assume role """
+    __instance = None
+
+    @staticmethod
+    def get_instance():
+        """ Static access method. """
+        if not AssumeRole.__instance:
+            AssumeRole()
+
+        return AssumeRole.__instance
+
+    def __init__(self):
+        """ Virtually private constructor. """
+        if AssumeRole.__instance:
+            raise Exception("This is a singleton class!")
+
+        sts = STSConnection(
+            settings.AWS_ACCESS_KEY_ID,
+            settings.AWS_SECRET_ACCESS_KEY
+        )
+        self.credentials = sts.assume_role(
+            role_arn=settings.ROLE_ARN,
+            role_session_name='vem',
+            duration_seconds=3600,
+            mfa_serial_number=settings.MFA_SERIAL_NUMBER,
+            mfa_token=settings.MFA_TOKEN
+        ).credentials.to_dict()
+
+        AssumeRole.__instance = self
+
+
 class TranscriptProvider(object):
     """
     Transcription Provider Enumeration
@@ -271,7 +308,7 @@ def validate_transcript_preferences(provider, cielo24_fidelity, cielo24_turnarou
                     return error, preferences
 
                 if not preferred_languages or not set(preferred_languages) <= set(supported_languages.keys()):
-                    error = 'Invalid languages {}.'.format(preferred_languages)  # pylint: disable=unicode-format-string
+                    error = 'Invalid languages {}.'.format(preferred_languages)
                     return error, preferences
 
                 # Validated Cielo24 preferences
@@ -765,10 +802,20 @@ def storage_service_bucket():
     """
     Returns an S3 bucket for video uploads.
     """
-    conn = s3.connection.S3Connection(
-        settings.AWS_ACCESS_KEY_ID,
-        settings.AWS_SECRET_ACCESS_KEY
-    )
+    if waffle_flags()[ENABLE_DEVSTACK_VIDEO_UPLOADS].is_enabled():
+        credentials = AssumeRole.get_instance().credentials
+        params = {
+            'aws_access_key_id': credentials['access_key'],
+            'aws_secret_access_key': credentials['secret_key'],
+            'security_token': credentials['session_token']
+        }
+    else:
+        params = {
+            'aws_access_key_id': settings.AWS_ACCESS_KEY_ID,
+            'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY
+        }
+
+    conn = s3.connection.S3Connection(**params)
     # We don't need to validate our bucket, it requires a very permissive IAM permission
     # set since behind the scenes it fires a HEAD request that is equivalent to get_all_keys()
     # meaning it would need ListObjects on the whole bucket, not just the path used in each

openedx/core/djangoapps/video_pipeline/config/waffle.py

@@ -3,13 +3,14 @@ This module contains configuration settings via waffle flags
 for the Video Pipeline app.
 """
 
-from openedx.core.djangoapps.waffle_utils import WaffleFlagNamespace, CourseWaffleFlag
+from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag, WaffleFlag, WaffleFlagNamespace
 
 # Videos Namespace
 WAFFLE_NAMESPACE = 'videos'
 
 # Waffle flag telling whether youtube is deprecated.
 DEPRECATE_YOUTUBE = 'deprecate_youtube'
+ENABLE_DEVSTACK_VIDEO_UPLOADS = 'enable_devstack_video_uploads'
 
 
 def waffle_flags():
@@ -21,5 +22,10 @@ def waffle_flags():
         DEPRECATE_YOUTUBE: CourseWaffleFlag(
             waffle_namespace=namespace,
             flag_name=DEPRECATE_YOUTUBE
+        ),
+        ENABLE_DEVSTACK_VIDEO_UPLOADS: WaffleFlag(
+            waffle_namespace=namespace,
+            flag_name=ENABLE_DEVSTACK_VIDEO_UPLOADS,
+            flag_undefined_default=False
         )
     }