From e4befb19262d1ff6557eb0bf1ed44df2a7bfcb3a Mon Sep 17 00:00:00 2001 From: Alan Zarembok Date: Tue, 3 Nov 2020 12:34:46 -0500 Subject: [PATCH] Add new generate_video_upload_link api. --- .../contentstore/views/tests/test_videos.py | 275 ++++++++++-------- cms/djangoapps/contentstore/views/videos.py | 34 ++- cms/urls.py | 2 + 3 files changed, 183 insertions(+), 128 deletions(-) diff --git a/cms/djangoapps/contentstore/views/tests/test_videos.py b/cms/djangoapps/contentstore/views/tests/test_videos.py index cdb9548adf..e608c0b495 100644 --- a/cms/djangoapps/contentstore/views/tests/test_videos.py +++ b/cms/djangoapps/contentstore/views/tests/test_videos.py @@ -206,12 +206,138 @@ class VideoUploadTestMixin(VideoUploadTestBase): self.assertEqual(self.client.get(self.url).status_code, 404) +class VideoUploadPostTestsMixin(object): + """ + Shared test cases for video post tests. + """ + @override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret') + @patch('boto.s3.key.Key') + @patch('boto.s3.connection.S3Connection') + def test_post_success(self, mock_conn, mock_key): + files = [ + { + 'file_name': 'first.mp4', + 'content_type': 'video/mp4', + }, + { + 'file_name': 'second.mp4', + 'content_type': 'video/mp4', + }, + { + 'file_name': 'third.mov', + 'content_type': 'video/quicktime', + }, + { + 'file_name': 'fourth.mp4', + 'content_type': 'video/mp4', + }, + ] + + bucket = Mock() + mock_conn.return_value = Mock(get_bucket=Mock(return_value=bucket)) + mock_key_instances = [ + Mock( + generate_url=Mock( + return_value='http://example.com/url_{}'.format(file_info['file_name']) + ) + ) + for file_info in files + ] + # If extra calls are made, return a dummy + mock_key.side_effect = mock_key_instances + [Mock()] + + response = self.client.post( + self.url, + json.dumps({'files': files}), + content_type='application/json' + ) + self.assertEqual(response.status_code, 200) + response_obj = json.loads(response.content.decode('utf-8')) + + mock_conn.assert_called_once_with( + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY + ) + self.assertEqual(len(response_obj['files']), len(files)) + self.assertEqual(mock_key.call_count, len(files)) + for i, file_info in enumerate(files): + # Ensure Key was set up correctly and extract id + key_call_args, __ = mock_key.call_args_list[i] + self.assertEqual(key_call_args[0], bucket) + path_match = re.match( + ( + settings.VIDEO_UPLOAD_PIPELINE['ROOT_PATH'] + + '/([a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12})$' + ), + key_call_args[1] + ) + self.assertIsNotNone(path_match) + video_id = path_match.group(1) + mock_key_instance = mock_key_instances[i] + + mock_key_instance.set_metadata.assert_any_call( + 'course_video_upload_token', + self.test_token + ) + + mock_key_instance.set_metadata.assert_any_call( + 'client_video_id', + file_info['file_name'] + ) + mock_key_instance.set_metadata.assert_any_call('course_key', six.text_type(self.course.id)) + mock_key_instance.generate_url.assert_called_once_with( + KEY_EXPIRATION_IN_SECONDS, + 'PUT', + headers={'Content-Type': file_info['content_type']} + ) + + # Ensure VAL was updated + val_info = get_video_info(video_id) + self.assertEqual(val_info['status'], 'upload') + self.assertEqual(val_info['client_video_id'], file_info['file_name']) + self.assertEqual(val_info['status'], 'upload') + self.assertEqual(val_info['duration'], 0) + self.assertEqual(val_info['courses'], [{six.text_type(self.course.id): None}]) + + # Ensure response is correct + response_file = response_obj['files'][i] + self.assertEqual(response_file['file_name'], file_info['file_name']) + self.assertEqual(response_file['upload_url'], mock_key_instance.generate_url()) + + def test_post_non_json(self): + response = self.client.post(self.url, {"files": []}) + self.assertEqual(response.status_code, 400) + + def test_post_malformed_json(self): + response = self.client.post(self.url, "{", content_type="application/json") + self.assertEqual(response.status_code, 400) + + def test_post_invalid_json(self): + def assert_bad(content): + """Make request with content and assert that response is 400""" + response = self.client.post( + self.url, + json.dumps(content), + content_type="application/json" + ) + self.assertEqual(response.status_code, 400) + + # Top level missing files key + assert_bad({}) + + # Entry missing file_name + assert_bad({"files": [{"content_type": "video/mp4"}]}) + + # Entry missing content_type + assert_bad({"files": [{"file_name": "test.mp4"}]}) + + @ddt.ddt @patch.dict("django.conf.settings.FEATURES", {"ENABLE_VIDEO_UPLOAD_PIPELINE": True}) @override_settings(VIDEO_UPLOAD_PIPELINE={ "VEM_S3_BUCKET": "vem_test_bucket", "BUCKET": "test_bucket", "ROOT_PATH": "test_root" }) -class VideosHandlerTestCase(VideoUploadTestMixin, CourseTestCase): +class VideosHandlerTestCase(VideoUploadTestMixin, VideoUploadPostTestsMixin, CourseTestCase): """Test cases for the main video upload endpoint""" VIEW_NAME = 'videos_handler' @@ -334,33 +460,6 @@ class VideosHandlerTestCase(VideoUploadTestMixin, CourseTestCase): self.assertEqual(response.status_code, 200) self.assertContains(response, 'video_upload_pagination') - def test_post_non_json(self): - response = self.client.post(self.url, {"files": []}) - self.assertEqual(response.status_code, 400) - - def test_post_malformed_json(self): - response = self.client.post(self.url, "{", content_type="application/json") - self.assertEqual(response.status_code, 400) - - def test_post_invalid_json(self): - def assert_bad(content): - """Make request with content and assert that response is 400""" - response = self.client.post( - self.url, - json.dumps(content), - content_type="application/json" - ) - self.assertEqual(response.status_code, 400) - - # Top level missing files key - assert_bad({}) - - # Entry missing file_name - assert_bad({"files": [{"content_type": "video/mp4"}]}) - - # Entry missing content_type - assert_bad({"files": [{"file_name": "test.mp4"}]}) - @override_settings(AWS_ACCESS_KEY_ID="test_key_id", AWS_SECRET_ACCESS_KEY="test_secret") @patch("boto.s3.key.Key") @patch("boto.s3.connection.S3Connection") @@ -502,100 +601,6 @@ class VideosHandlerTestCase(VideoUploadTestMixin, CourseTestCase): settings.VIDEO_UPLOAD_PIPELINE['VEM_S3_BUCKET'], validate=False # pylint: disable=unsubscriptable-object ) - @override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret') - @patch('boto.s3.key.Key') - @patch('boto.s3.connection.S3Connection') - def test_post_success(self, mock_conn, mock_key): - files = [ - { - 'file_name': 'first.mp4', - 'content_type': 'video/mp4', - }, - { - 'file_name': 'second.mp4', - 'content_type': 'video/mp4', - }, - { - 'file_name': 'third.mov', - 'content_type': 'video/quicktime', - }, - { - 'file_name': 'fourth.mp4', - 'content_type': 'video/mp4', - }, - ] - - bucket = Mock() - mock_conn.return_value = Mock(get_bucket=Mock(return_value=bucket)) - mock_key_instances = [ - Mock( - generate_url=Mock( - return_value='http://example.com/url_{}'.format(file_info['file_name']) - ) - ) - for file_info in files - ] - # If extra calls are made, return a dummy - mock_key.side_effect = mock_key_instances + [Mock()] - - response = self.client.post( - self.url, - json.dumps({'files': files}), - content_type='application/json' - ) - self.assertEqual(response.status_code, 200) - response_obj = json.loads(response.content.decode('utf-8')) - - mock_conn.assert_called_once_with( - aws_access_key_id=settings.AWS_ACCESS_KEY_ID, - aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY - ) - self.assertEqual(len(response_obj['files']), len(files)) - self.assertEqual(mock_key.call_count, len(files)) - for i, file_info in enumerate(files): - # Ensure Key was set up correctly and extract id - key_call_args, __ = mock_key.call_args_list[i] - self.assertEqual(key_call_args[0], bucket) - path_match = re.match( - ( - settings.VIDEO_UPLOAD_PIPELINE['ROOT_PATH'] + - '/([a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12})$' - ), - key_call_args[1] - ) - self.assertIsNotNone(path_match) - video_id = path_match.group(1) - mock_key_instance = mock_key_instances[i] - - mock_key_instance.set_metadata.assert_any_call( - 'course_video_upload_token', - self.test_token - ) - - mock_key_instance.set_metadata.assert_any_call( - 'client_video_id', - file_info['file_name'] - ) - mock_key_instance.set_metadata.assert_any_call('course_key', six.text_type(self.course.id)) - mock_key_instance.generate_url.assert_called_once_with( - KEY_EXPIRATION_IN_SECONDS, - 'PUT', - headers={'Content-Type': file_info['content_type']} - ) - - # Ensure VAL was updated - val_info = get_video_info(video_id) - self.assertEqual(val_info['status'], 'upload') - self.assertEqual(val_info['client_video_id'], file_info['file_name']) - self.assertEqual(val_info['status'], 'upload') - self.assertEqual(val_info['duration'], 0) - self.assertEqual(val_info['courses'], [{six.text_type(self.course.id): None}]) - - # Ensure response is correct - response_file = response_obj['files'][i] - self.assertEqual(response_file['file_name'], file_info['file_name']) - self.assertEqual(response_file['upload_url'], mock_key_instance.generate_url()) - @override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret') @patch('boto.s3.key.Key') @patch('boto.s3.connection.S3Connection') @@ -840,6 +845,32 @@ class VideosHandlerTestCase(VideoUploadTestMixin, CourseTestCase): self.assertNotContains(response, button_html) +@ddt.ddt +@patch.dict("django.conf.settings.FEATURES", {"ENABLE_VIDEO_UPLOAD_PIPELINE": True}) +@override_settings(VIDEO_UPLOAD_PIPELINE={ + "VEM_S3_BUCKET": "vem_test_bucket", "BUCKET": "test_bucket", "ROOT_PATH": "test_root" +}) +class GenerateVideoUploadLinkTestCase(VideoUploadTestBase, VideoUploadPostTestsMixin, CourseTestCase): + """ + Test cases for the main video upload endpoint + """ + + VIEW_NAME = 'generate_video_upload_link' + + def test_unsupported_requests_fail(self): + """ + The API only supports post, make sure other requests fail + """ + response = self.client.get(self.url) + self.assertEqual(response.status_code, 405) + + response = self.client.put(self.url) + self.assertEqual(response.status_code, 405) + + response = self.client.patch(self.url) + self.assertEqual(response.status_code, 405) + + @ddt.ddt @patch.dict('django.conf.settings.FEATURES', {'ENABLE_VIDEO_UPLOAD_PIPELINE': True}) @override_settings(VIDEO_UPLOAD_PIPELINE={'BUCKET': 'test_bucket', 'ROOT_PATH': 'test_root'}) diff --git a/cms/djangoapps/contentstore/views/videos.py b/cms/djangoapps/contentstore/views/videos.py index ea5111169d..d00b9d4639 100644 --- a/cms/djangoapps/contentstore/views/videos.py +++ b/cms/djangoapps/contentstore/views/videos.py @@ -40,6 +40,9 @@ from edxval.api import ( ) from opaque_keys.edx.keys import CourseKey from pytz import UTC +from rest_framework import status as rest_status +from rest_framework.decorators import api_view +from rest_framework.response import Response from edx_toggles.toggles import WaffleFlagNamespace, WaffleSwitchNamespace from edxmako.shortcuts import render_to_response @@ -50,6 +53,7 @@ from openedx.core.djangoapps.video_pipeline.config.waffle import ( waffle_flags ) from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag +from openedx.core.lib.api.view_utils import view_auth_classes from util.json_request import JsonResponse, expect_json from xmodule.video_module.transcripts_utils import Transcript @@ -63,6 +67,7 @@ __all__ = [ 'video_encodings_download', 'video_images_handler', 'transcript_preferences_handler', + 'generate_video_upload_link_handler', ] LOGGER = logging.getLogger(__name__) @@ -215,7 +220,24 @@ def videos_handler(request, course_key_string, edx_video_id=None): elif _is_pagination_context_update_request(request): return _update_pagination_context(request) - return videos_post(course, request) + data, status = videos_post(course, request) + return JsonResponse(data, status=status) + + +@api_view(['POST']) +@view_auth_classes() +@expect_json +def generate_video_upload_link_handler(request, course_key_string): + """ + API for creating a video upload. Returns an edx_video_id and a presigned URL that can be used + to upload the video to AWS S3. + """ + course = _get_and_validate_course(course_key_string, request.user) + if not course: + return Response(data='Course Not Found', status=rest_status.HTTP_400_BAD_REQUEST) + + data, status = videos_post(course, request) + return Response(data, status=status) @expect_json @@ -714,9 +736,9 @@ def videos_post(course, request): error = "Request 'files' entry contain unsupported content_type" if error: - return JsonResponse({'error': error}, status=400) + return {'error': error}, 400 - bucket = storage_service_bucket(course.id) + bucket = storage_service_bucket() req_files = data['files'] resp_files = [] @@ -727,7 +749,7 @@ def videos_post(course, request): file_name.encode('ascii') except UnicodeEncodeError: error_msg = u'The file name for %s must contain only ASCII characters.' % file_name - return JsonResponse({'error': error_msg}, status=400) + return {'error': error_msg}, 400 edx_video_id = six.text_type(uuid4()) key = storage_service_key(bucket, file_name=edx_video_id) @@ -771,10 +793,10 @@ def videos_post(course, request): resp_files.append({'file_name': file_name, 'upload_url': upload_url, 'edx_video_id': edx_video_id}) - return JsonResponse({'files': resp_files}, status=200) + return {'files': resp_files}, 200 -def storage_service_bucket(course_key=None): +def storage_service_bucket(): """ Returns an S3 bucket for video upload. """ diff --git a/cms/urls.py b/cms/urls.py index bb51c55f85..0fc4f44351 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -154,6 +154,8 @@ urlpatterns = [ contentstore_views.textbooks_detail_handler, name='textbooks_detail_handler'), url(r'^videos/{}(?:/(?P[-\w]+))?$'.format(settings.COURSE_KEY_PATTERN), contentstore_views.videos_handler, name='videos_handler'), + url(r'^generate_video_upload_link/{}'.format(settings.COURSE_KEY_PATTERN), + contentstore_views.generate_video_upload_link_handler, name='generate_video_upload_link'), url(r'^video_images/{}(?:/(?P[-\w]+))?$'.format(settings.COURSE_KEY_PATTERN), contentstore_views.video_images_handler, name='video_images_handler'), url(r'^transcript_preferences/{}$'.format(settings.COURSE_KEY_PATTERN),