Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Studio content api videos (#32803)

Browse Source 
* refactor: extract methods to video_storage_handlers

* refactor: move private functions

* refactor: move functions to videos_storage_handlers

* refactor: asset_storage_handlers

* feat: add video api views

* feat: add video urls

* feat: add mock videos post

* refactor: mock video upload url

* fix: json extraction

* fix: url pattern for video deletion

* fix: video url views

* fix: lint

* fix: lint

* fix: tests

* fix: tests

* fix: tests

* Feat  studio content api transcripts (#32858)

* feat: add transcript endpoints

feat: add transcript upload endpoint, check that transcripts for deletion exist

fix: remove transcript credentials view cause out of scope

* fix: lint

* feat: TNL-10897 fix destroy() args to kwargs bug

---------

Co-authored-by: Bernard Szabo <bszabo@edx.org>

---------

Co-authored-by: Bernard Szabo <bszabo@edx.org>
This commit is contained in:
Jesper Hodge
2023-07-31 13:37:00 -04:00
committed by GitHub
parent 96699d577c
commit 6598abbb6b
15 changed files with 1551 additions and 959 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
cms/djangoapps/contentstore/rest_api/v1/urls.py
View File

@@ -13,10 +13,14 @@ from .views import (
ProctoringErrorsView,
xblock,
assets,
videos,
transcripts,
)
app_name = 'v1'
VIDEO_ID_PATTERN = r'(?:(?P<edx_video_id>[-\w]+))'
urlpatterns = [
re_path(
fr'^proctored_exam_settings/{COURSE_ID_PATTERN}$',
@@ -51,4 +55,28 @@ urlpatterns = [
fr'^file_assets/{settings.COURSE_ID_PATTERN}/{settings.ASSET_KEY_PATTERN}?$',
assets.AssetsView.as_view(), name='studio_content_assets'
),
re_path(
fr'^videos/uploads/{settings.COURSE_ID_PATTERN}/{VIDEO_ID_PATTERN}?$',
videos.VideosView.as_view(), name='studio_content_videos_uploads'
),
re_path(
fr'^videos/images/{settings.COURSE_ID_PATTERN}/{VIDEO_ID_PATTERN}?$',
videos.VideoImagesView.as_view(), name='studio_content_videos_images'
),
re_path(
fr'^videos/encodings/{settings.COURSE_ID_PATTERN}$',
videos.VideoEncodingsDownloadView.as_view(), name='studio_content_videos_encodings'
),
re_path(
r'^videos/features/$',
videos.VideoFeaturesView.as_view(), name='studio_content_videos_features'
),
re_path(
fr'^videos/upload_link/{settings.COURSE_ID_PATTERN}$',
videos.UploadLinkView.as_view(), name='studio_content_videos_upload_link'
),
re_path(
fr'^video_transcripts/{settings.COURSE_ID_PATTERN}$',
transcripts.TranscriptView.as_view(), name='studio_content_video_transcripts'
),
]

2
cms/djangoapps/contentstore/rest_api/v1/views/__init__.py
View File

@@ -6,3 +6,5 @@ from .grading import CourseGradingView
from .proctoring import ProctoredExamSettingsView, ProctoringErrorsView
from .settings import CourseSettingsView
from .xblock import XblockView
from .assets import AssetsView
from .videos import VideosView

6
cms/djangoapps/contentstore/rest_api/v1/views/assets.py
View File

@@ -1,4 +1,6 @@
# lint-amnesty, pylint: disable=missing-module-docstring
"""
Public rest API endpoints for the Studio Content API Assets.
"""
import logging
from rest_framework.generics import RetrieveUpdateDestroyAPIView, CreateAPIView
from django.views.decorators.csrf import csrf_exempt
@@ -19,7 +21,7 @@ toggles = contentstore_toggles
@view_auth_classes()
class AssetsView(DeveloperErrorViewMixin, RetrieveUpdateDestroyAPIView, CreateAPIView):
"""
public rest API endpoint for the Studio Content API.
public rest API endpoints for the Studio Content API Assets.
course_key: required argument, needed to authorize course authors and identify the asset.
asset_key_string: required argument, needed to identify the asset.
"""

62
cms/djangoapps/contentstore/rest_api/v1/views/transcripts.py Normal file
View File

@@ -0,0 +1,62 @@
"""
Public rest API endpoints for the Studio Content API video assets.
"""
import logging
from rest_framework.generics import (
CreateAPIView,
RetrieveAPIView,
DestroyAPIView
)
from django.views.decorators.csrf import csrf_exempt
from django.http import Http404
from openedx.core.lib.api.view_utils import DeveloperErrorViewMixin, view_auth_classes
from common.djangoapps.util.json_request import expect_json_in_class_view
from ....api import course_author_access_required
from cms.djangoapps.contentstore.transcript_storage_handlers import (
upload_transcript,
delete_video_transcript_or_404,
handle_transcript_download,
)
import cms.djangoapps.contentstore.toggles as contentstore_toggles
log = logging.getLogger(__name__)
toggles = contentstore_toggles
@view_auth_classes()
class TranscriptView(DeveloperErrorViewMixin, CreateAPIView, RetrieveAPIView, DestroyAPIView):
"""
public rest API endpoints for the Studio Content API video transcripts.
course_key: required argument, needed to authorize course authors and identify the video.
edx_video_id: optional query parameter, needed to identify the transcript.
language_code: optional query parameter, needed to identify the transcript.
"""
def dispatch(self, request, *args, **kwargs):
if not toggles.use_studio_content_api():
raise Http404
return super().dispatch(request, *args, **kwargs)
@csrf_exempt
@course_author_access_required
@expect_json_in_class_view
def create(self, request, course_key_string): # pylint: disable=arguments-differ
return upload_transcript(request)
@course_author_access_required
def retrieve(self, request, course_key_string): # pylint: disable=arguments-differ
"""
Get a video transcript. edx_video_id and language_code query parameters are required.
"""
return handle_transcript_download(request)
@course_author_access_required
def destroy(self, request, course_key_string): # pylint: disable=arguments-differ
"""
Delete a video transcript. edx_video_id and language_code query parameters are required.
"""
return delete_video_transcript_or_404(request)

159
cms/djangoapps/contentstore/rest_api/v1/views/videos.py Normal file
View File

@@ -0,0 +1,159 @@
"""
Public rest API endpoints for the Studio Content API video assets.
"""
import logging
from rest_framework.generics import (
CreateAPIView,
RetrieveAPIView,
DestroyAPIView
)
from django.views.decorators.csrf import csrf_exempt
from django.http import Http404
from openedx.core.lib.api.view_utils import DeveloperErrorViewMixin, view_auth_classes
from common.djangoapps.util.json_request import expect_json_in_class_view
from ....api import course_author_access_required
from cms.djangoapps.contentstore.video_storage_handlers import (
handle_videos,
get_video_encodings_download,
handle_video_images,
enabled_video_features,
handle_generate_video_upload_link
)
import cms.djangoapps.contentstore.toggles as contentstore_toggles
log = logging.getLogger(__name__)
toggles = contentstore_toggles
@view_auth_classes()
class VideosView(DeveloperErrorViewMixin, CreateAPIView, RetrieveAPIView, DestroyAPIView):
"""
public rest API endpoints for the Studio Content API video assets.
course_key: required argument, needed to authorize course authors and identify the video.
video_id: required argument, needed to identify the video.
"""
def dispatch(self, request, *args, **kwargs):
# TODO: probably want to refactor this to a decorator.
"""
The dispatch method of a View class handles HTTP requests in general
and calls other methods to handle specific HTTP methods.
We use this to raise a 404 if the content api is disabled.
"""
if not toggles.use_studio_content_api():
raise Http404
return super().dispatch(request, *args, **kwargs)
@csrf_exempt
@course_author_access_required
@expect_json_in_class_view
def create(self, request, course_key): # pylint: disable=arguments-differ
return handle_videos(request, course_key.html_id())
@course_author_access_required
def retrieve(self, request, course_key, edx_video_id=None): # pylint: disable=arguments-differ
return handle_videos(request, course_key.html_id(), edx_video_id)
@course_author_access_required
@expect_json_in_class_view
def destroy(self, request, course_key, edx_video_id): # pylint: disable=arguments-differ
return handle_videos(request, course_key.html_id(), edx_video_id)
@view_auth_classes()
class VideoImagesView(DeveloperErrorViewMixin, CreateAPIView):
"""
public rest API endpoint for uploading a video image.
course_key: required argument, needed to authorize course authors and identify the video.
video_id: required argument, needed to identify the video.
"""
def dispatch(self, request, *args, **kwargs):
# TODO: probably want to refactor this to a decorator.
"""
The dispatch method of a View class handles HTTP requests in general
and calls other methods to handle specific HTTP methods.
We use this to raise a 404 if the content api is disabled.
"""
if not toggles.use_studio_content_api():
raise Http404
return super().dispatch(request, *args, **kwargs)
@csrf_exempt
@course_author_access_required
@expect_json_in_class_view
def create(self, request, course_key, edx_video_id=None): # pylint: disable=arguments-differ
return handle_video_images(request, course_key.html_id(), edx_video_id)
@view_auth_classes()
class VideoEncodingsDownloadView(DeveloperErrorViewMixin, RetrieveAPIView):
"""
public rest API endpoint providing a CSV report containing the encoded video URLs for video uploads.
course_key: required argument, needed to authorize course authors and identify relevant videos.
"""
def dispatch(self, request, *args, **kwargs):
# TODO: probably want to refactor this to a decorator.
"""
The dispatch method of a View class handles HTTP requests in general
and calls other methods to handle specific HTTP methods.
We use this to raise a 404 if the content api is disabled.
"""
if not toggles.use_studio_content_api():
raise Http404
return super().dispatch(request, *args, **kwargs)
@csrf_exempt
@course_author_access_required
def retrieve(self, request, course_key): # pylint: disable=arguments-differ
return get_video_encodings_download(request, course_key.html_id())
@view_auth_classes()
class VideoFeaturesView(DeveloperErrorViewMixin, RetrieveAPIView):
"""
public rest API endpoint providing a list of enabled video features.
"""
def dispatch(self, request, *args, **kwargs):
# TODO: probably want to refactor this to a decorator.
"""
The dispatch method of a View class handles HTTP requests in general
and calls other methods to handle specific HTTP methods.
We use this to raise a 404 if the content api is disabled.
"""
if not toggles.use_studio_content_api():
raise Http404
return super().dispatch(request, *args, **kwargs)
@csrf_exempt
def retrieve(self, request): # pylint: disable=arguments-differ
return enabled_video_features(request)
@view_auth_classes()
class UploadLinkView(DeveloperErrorViewMixin, CreateAPIView):
"""
public rest API endpoint providing a list of enabled video features.
"""
def dispatch(self, request, *args, **kwargs):
# TODO: probably want to refactor this to a decorator.
"""
The dispatch method of a View class handles HTTP requests in general
and calls other methods to handle specific HTTP methods.
We use this to raise a 404 if the content api is disabled.
"""
if not toggles.use_studio_content_api():
raise Http404
return super().dispatch(request, *args, **kwargs)
@csrf_exempt
@course_author_access_required
@expect_json_in_class_view
def create(self, request, course_key): # pylint: disable=arguments-differ
return handle_generate_video_upload_link(request, course_key.html_id())

6
cms/djangoapps/contentstore/rest_api/v1/views/xblock.py
View File

@@ -1,4 +1,6 @@
# lint-amnesty, pylint: disable=missing-module-docstring
"""
Public rest API endpoints for the Studio Content API.
"""
import logging
from rest_framework.generics import RetrieveUpdateDestroyAPIView, CreateAPIView
from django.views.decorators.csrf import csrf_exempt
@@ -20,7 +22,7 @@ handle_xblock = view_handlers.handle_xblock
@view_auth_classes()
class XblockView(DeveloperErrorViewMixin, RetrieveUpdateDestroyAPIView, CreateAPIView):
"""
public rest API endpoint for the Studio Content API.
Public rest API endpoints for the Studio Content API.
course_key: required argument, needed to authorize course authors.
usage_key_string (optional):
xblock identifier, for example in the form of "block-v1:<course id>+type@<type>+block@<block id>"

19
cms/djangoapps/contentstore/toggles.py
View File

@@ -497,6 +497,25 @@ def use_new_course_team_page(course_key):
return ENABLE_NEW_STUDIO_COURSE_TEAM_PAGE.is_enabled(course_key)
# .. toggle_name: contentstore.mock_video_uploads
# .. toggle_implementation: WaffleFlag
# .. toggle_default: False
# .. toggle_description: This flag mocks contentstore video uploads for local development, if you don't have access to AWS
# .. toggle_use_cases: open_edx
# .. toggle_creation_date: 2023-7-25
# .. toggle_tickets: TNL-10897
# .. toggle_warning:
MOCK_VIDEO_UPLOADS = WaffleFlag(
f'{CONTENTSTORE_NAMESPACE}.mock_video_uploads', __name__)
def use_mock_video_uploads():
"""
Returns a boolean if video uploads should be mocked for local development
"""
return MOCK_VIDEO_UPLOADS.is_enabled()
# .. toggle_name: contentstore.default_enable_flexible_peer_openassessments
# .. toggle_implementation: CourseWaffleFlag
# .. toggle_default: False

265
cms/djangoapps/contentstore/transcript_storage_handlers.py Normal file
View File

@@ -0,0 +1,265 @@
"""
Business logic for video transcripts.
"""
import logging
import os
from django.core.files.base import ContentFile
from django.http import HttpResponse, HttpResponseNotFound
from django.utils.translation import gettext as _
from edxval.api import (
create_or_update_video_transcript,
delete_video_transcript as delete_video_transcript_source_function,
get_3rd_party_transcription_plans,
get_available_transcript_languages,
get_video_transcript_data,
update_transcript_credentials_state_for_org,
get_video_transcript
)
from opaque_keys.edx.keys import CourseKey
from common.djangoapps.util.json_request import JsonResponse
from openedx.core.djangoapps.video_config.models import VideoTranscriptEnabledFlag
from openedx.core.djangoapps.video_pipeline.api import update_3rd_party_transcription_service_credentials
from xmodule.video_block.transcripts_utils import Transcript, TranscriptsGenerationException # lint-amnesty, pylint: disable=wrong-import-order
from .toggles import use_mock_video_uploads
from .video_storage_handlers import TranscriptProvider
LOGGER = logging.getLogger(__name__)
class TranscriptionProviderErrorType:
"""
Transcription provider's error types enumeration.
"""
INVALID_CREDENTIALS = 1
def validate_transcript_credentials(provider, **credentials):
"""
Validates transcript credentials.
Validations:
Providers must be either 3PlayMedia or Cielo24.
In case of:
3PlayMedia - 'api_key' and 'api_secret_key' are required.
Cielo24 - 'api_key' and 'username' are required.
It ignores any extra/unrelated parameters passed in credentials and
only returns the validated ones.
"""
error_message, validated_credentials = '', {}
valid_providers = list(get_3rd_party_transcription_plans().keys())
if provider in valid_providers:
must_have_props = []
if provider == TranscriptProvider.THREE_PLAY_MEDIA:
must_have_props = ['api_key', 'api_secret_key']
elif provider == TranscriptProvider.CIELO24:
must_have_props = ['api_key', 'username']
missing = [
must_have_prop for must_have_prop in must_have_props if must_have_prop not in list(credentials.keys()) # lint-amnesty, pylint: disable=consider-iterating-dictionary
]
if missing:
error_message = '{missing} must be specified.'.format(missing=' and '.join(missing))
return error_message, validated_credentials
validated_credentials.update({
prop: credentials[prop] for prop in must_have_props
})
else:
error_message = f'Invalid Provider {provider}.'
return error_message, validated_credentials
def handle_transcript_credentials(request, course_key_string):
"""
JSON view handler to update the transcript organization credentials.
Arguments:
request: WSGI request object
course_key_string: A course identifier to extract the org.
Returns:
- A 200 response if credentials are valid and successfully updated in edx-video-pipeline.
- A 404 response if transcript feature is not enabled for this course.
- A 400 if credentials do not pass validations, hence not updated in edx-video-pipeline.
"""
course_key = CourseKey.from_string(course_key_string)
if not VideoTranscriptEnabledFlag.feature_enabled(course_key):
return HttpResponseNotFound()
provider = request.json.pop('provider')
error_message, validated_credentials = validate_transcript_credentials(provider=provider, **request.json)
if error_message:
response = JsonResponse({'error': error_message}, status=400)
else:
# Send the validated credentials to edx-video-pipeline and video-encode-manager
credentials_payload = dict(validated_credentials, org=course_key.org, provider=provider)
error_response, is_updated = update_3rd_party_transcription_service_credentials(**credentials_payload)
# Send appropriate response based on whether credentials were updated or not.
if is_updated:
# Cache credentials state in edx-val.
update_transcript_credentials_state_for_org(org=course_key.org, provider=provider, exists=is_updated)
response = JsonResponse(status=200)
else:
# Error response would contain error types and the following
# error type is received from edx-video-pipeline whenever we've
# got invalid credentials for a provider. Its kept this way because
# edx-video-pipeline doesn't support i18n translations yet.
error_type = error_response.get('error_type')
if error_type == TranscriptionProviderErrorType.INVALID_CREDENTIALS:
error_message = _('The information you entered is incorrect.')
response = JsonResponse({'error': error_message}, status=400)
return response
def handle_transcript_download(request):
"""
JSON view handler to download a transcript.
Arguments:
request: WSGI request object
Returns:
- A 200 response with SRT transcript file attached.
- A 400 if there is a validation error.
- A 404 if there is no such transcript.
"""
missing = [attr for attr in ['edx_video_id', 'language_code'] if attr not in request.GET]
if missing:
return JsonResponse(
{'error': _('The following parameters are required: {missing}.').format(missing=', '.join(missing))},
status=400
)
edx_video_id = request.GET['edx_video_id']
language_code = request.GET['language_code']
transcript = get_video_transcript_data(video_id=edx_video_id, language_code=language_code)
if transcript:
name_and_extension = os.path.splitext(transcript['file_name'])
basename, file_format = name_and_extension[0], name_and_extension[1][1:]
transcript_filename = f'{basename}.{Transcript.SRT}'
transcript_content = Transcript.convert(
content=transcript['content'],
input_format=file_format,
output_format=Transcript.SRT
)
# Construct an HTTP response
response = HttpResponse(transcript_content, content_type=Transcript.mime_types[Transcript.SRT])
response['Content-Disposition'] = f'attachment; filename="{transcript_filename}"'
else:
response = HttpResponseNotFound()
return response
def _create_or_update_video_transcript(**kwargs):
if use_mock_video_uploads():
return True
return create_or_update_video_transcript(**kwargs)
def upload_transcript(request):
"""
Upload a transcript file
Arguments:
request: A WSGI request object
Transcript file in SRT format
"""
edx_video_id = request.POST['edx_video_id']
language_code = request.POST['language_code']
new_language_code = request.POST['new_language_code']
transcript_file = request.FILES['file']
try:
# Convert SRT transcript into an SJSON format
# and upload it to S3.
sjson_subs = Transcript.convert(
content=transcript_file.read().decode('utf-8'),
input_format=Transcript.SRT,
output_format=Transcript.SJSON
).encode()
_create_or_update_video_transcript(
video_id=edx_video_id,
language_code=language_code,
metadata={
'provider': TranscriptProvider.CUSTOM,
'file_format': Transcript.SJSON,
'language_code': new_language_code
},
file_data=ContentFile(sjson_subs),
)
response = JsonResponse(status=201)
except (TranscriptsGenerationException, UnicodeDecodeError):
LOGGER.error("Unable to update transcript on edX video %s for language %s", edx_video_id, new_language_code)
response = JsonResponse(
{'error': _('There is a problem with this transcript file. Try to upload a different file.')},
status=400
)
finally:
LOGGER.info("Updated transcript on edX video %s for language %s", edx_video_id, new_language_code)
return response
def validate_transcript_upload_data(data, files):
"""
Validates video transcript file.
Arguments:
data: A request's data part.
files: A request's files part.
Returns:
None or String
If there is error returns error message otherwise None.
"""
error = None
# Validate the must have attributes - this error is unlikely to be faced by common users.
must_have_attrs = ['edx_video_id', 'language_code', 'new_language_code']
missing = [attr for attr in must_have_attrs if attr not in data]
if missing:
error = _('The following parameters are required: {missing}.').format(missing=', '.join(missing))
elif (
data['language_code'] != data['new_language_code'] and
data['new_language_code'] in get_available_transcript_languages(video_id=data['edx_video_id'])
):
error = _('A transcript with the "{language_code}" language code already exists.'.format( # lint-amnesty, pylint: disable=translation-of-non-string
language_code=data['new_language_code']
))
elif 'file' not in files:
error = _('A transcript file is required.')
return error
def delete_video_transcript(video_id=None, language_code=None):
return delete_video_transcript_source_function(video_id=video_id, language_code=language_code)
def delete_video_transcript_or_404(request):
"""
Delete a video transcript or return 404 if it doesn't exist.
"""
missing = [attr for attr in ['edx_video_id', 'language_code'] if attr not in request.GET]
if missing:
return JsonResponse(
{'error': _('The following parameters are required: {missing}.').format(missing=', '.join(missing))},
status=400
)
video_id = request.GET.get('edx_video_id')
language_code = request.GET.get('language_code')
if not get_video_transcript(video_id=video_id, language_code=language_code):
return HttpResponseNotFound()
delete_video_transcript(video_id=video_id, language_code=language_code)
return JsonResponse(status=200)

902
cms/djangoapps/contentstore/video_storage_handlers.py Normal file
View File

@@ -0,0 +1,902 @@
"""
Views related to the video upload feature
"""
import codecs
import csv
import io
import json
import logging
from contextlib import closing
from datetime import datetime, timedelta
from uuid import uuid4
from boto.s3.connection import S3Connection
from boto import s3
from django.conf import settings
from django.contrib.staticfiles.storage import staticfiles_storage
from django.http import FileResponse, HttpResponseNotFound
from django.shortcuts import redirect
from django.urls import reverse
from django.utils.translation import gettext as _
from django.utils.translation import gettext_noop
from edx_toggles.toggles import WaffleSwitch
from edxval.api import (
SortDirection,
VideoSortField,
create_or_update_transcript_preferences,
create_video,
get_3rd_party_transcription_plans,
get_available_transcript_languages,
get_video_transcript_url,
get_transcript_credentials_state_for_org,
get_transcript_preferences,
get_videos_for_course,
remove_transcript_preferences,
remove_video_for_course,
update_video_image,
update_video_status
)
from opaque_keys.edx.keys import CourseKey
from pytz import UTC
from rest_framework import status as rest_status
from rest_framework.response import Response
from common.djangoapps.edxmako.shortcuts import render_to_response
from common.djangoapps.util.json_request import JsonResponse
from openedx.core.djangoapps.video_config.models import VideoTranscriptEnabledFlag
from openedx.core.djangoapps.video_config.toggles import PUBLIC_VIDEO_SHARE
from openedx.core.djangoapps.video_pipeline.config.waffle import (
DEPRECATE_YOUTUBE,
ENABLE_DEVSTACK_VIDEO_UPLOADS,
)
from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag
from xmodule.video_block.transcripts_utils import Transcript # lint-amnesty, pylint: disable=wrong-import-order
from .models import VideoUploadConfig
from .toggles import use_new_video_uploads_page, use_mock_video_uploads
from .utils import reverse_course_url, get_video_uploads_url
from .video_utils import validate_video_image
from .views.course import get_course_and_check_access
LOGGER = logging.getLogger(__name__)
# Waffle switches namespace for videos
WAFFLE_NAMESPACE = 'videos'
# Waffle switch for enabling/disabling video image upload feature
VIDEO_IMAGE_UPLOAD_ENABLED = WaffleSwitch( # lint-amnesty, pylint: disable=toggle-missing-annotation
f'{WAFFLE_NAMESPACE}.video_image_upload_enabled', __name__
)
# Waffle flag namespace for studio
WAFFLE_STUDIO_FLAG_NAMESPACE = 'studio'
ENABLE_VIDEO_UPLOAD_PAGINATION = CourseWaffleFlag( # lint-amnesty, pylint: disable=toggle-missing-annotation
f'{WAFFLE_STUDIO_FLAG_NAMESPACE}.enable_video_upload_pagination', __name__
)
# Default expiration, in seconds, of one-time URLs used for uploading videos.
KEY_EXPIRATION_IN_SECONDS = 86400
VIDEO_SUPPORTED_FILE_FORMATS = {
'.mp4': 'video/mp4',
'.mov': 'video/quicktime',
}
VIDEO_UPLOAD_MAX_FILE_SIZE_GB = 5
# maximum time for video to remain in upload state
MAX_UPLOAD_HOURS = 24
VIDEOS_PER_PAGE = 100
class TranscriptProvider:
"""
Transcription Provider Enumeration
"""
CIELO24 = 'Cielo24'
THREE_PLAY_MEDIA = '3PlayMedia'
CUSTOM = 'Custom'
class StatusDisplayStrings:
"""
A class to map status strings as stored in VAL to display strings for the
video upload page
"""
# Translators: This is the status of an active video upload
_UPLOADING = gettext_noop("Uploading")
# Translators: This is the status for a video that the servers are currently processing
_IN_PROGRESS = gettext_noop("In Progress")
# Translators: This is the status for a video that the servers have successfully processed
_COMPLETE = gettext_noop("Ready")
# Translators: This is the status for a video that is uploaded completely
_UPLOAD_COMPLETED = gettext_noop("Uploaded")
# Translators: This is the status for a video that the servers have failed to process
_FAILED = gettext_noop("Failed")
# Translators: This is the status for a video that is cancelled during upload by user
_CANCELLED = gettext_noop("Cancelled")
# Translators: This is the status for a video which has failed
# due to being flagged as a duplicate by an external or internal CMS
_DUPLICATE = gettext_noop("Failed Duplicate")
# Translators: This is the status for a video which has duplicate token for youtube
_YOUTUBE_DUPLICATE = gettext_noop("YouTube Duplicate")
# Translators: This is the status for a video for which an invalid
# processing token was provided in the course settings
_INVALID_TOKEN = gettext_noop("Invalid Token")
# Translators: This is the status for a video that was included in a course import
_IMPORTED = gettext_noop("Imported")
# Translators: This is the status for a video that is in an unknown state
_UNKNOWN = gettext_noop("Unknown")
# Translators: This is the status for a video that is having its transcription in progress on servers
_TRANSCRIPTION_IN_PROGRESS = gettext_noop("Transcription in Progress")
# Translators: This is the status for a video whose transcription is complete
_TRANSCRIPT_READY = gettext_noop("Transcript Ready")
# Translators: This is the status for a video whose transcription job was failed for some languages
_PARTIAL_FAILURE = gettext_noop("Partial Failure")
# Translators: This is the status for a video whose transcription job has failed altogether
_TRANSCRIPT_FAILED = gettext_noop("Transcript Failed")
_STATUS_MAP = {
"upload": _UPLOADING,
"ingest": _IN_PROGRESS,
"transcode_queue": _IN_PROGRESS,
"transcode_active": _IN_PROGRESS,
"file_delivered": _COMPLETE,
"file_complete": _COMPLETE,
"upload_completed": _UPLOAD_COMPLETED,
"file_corrupt": _FAILED,
"pipeline_error": _FAILED,
"upload_failed": _FAILED,
"s3_upload_failed": _FAILED,
"upload_cancelled": _CANCELLED,
"duplicate": _DUPLICATE,
"youtube_duplicate": _YOUTUBE_DUPLICATE,
"invalid_token": _INVALID_TOKEN,
"imported": _IMPORTED,
"transcription_in_progress": _TRANSCRIPTION_IN_PROGRESS,
"transcript_ready": _TRANSCRIPT_READY,
"partial_failure": _PARTIAL_FAILURE,
# TODO: Add a related unit tests when the VAL update is part of platform
"transcript_failed": _TRANSCRIPT_FAILED,
}
@staticmethod
def get(val_status):
"""Map a VAL status string to a localized display string"""
# pylint: disable=translation-of-non-string
return _(StatusDisplayStrings._STATUS_MAP.get(val_status, StatusDisplayStrings._UNKNOWN))
def handle_videos(request, course_key_string, edx_video_id=None):
"""
Restful handler for video uploads.
GET
html: return an HTML page to display previous video uploads and allow
new ones
json: return json representing the videos that have been uploaded and
their statuses
POST
json: generate new video upload urls, for example upload urls for S3 buckets. To upload the video, you should
make a PUT request to the returned upload_url values. This can happen on the frontend, MFE,
or client side - it is not implemented in the backend.
Example payload:
{
"files": [{
"file_name": "video.mp4",
"content_type": "video/mp4"
}]
}
Returns (JSON):
{
"files": [{
"file_name": "video.mp4",
"upload_url": "http://example.com/put_video"
}]
}
DELETE
soft deletes a video for particular course
"""
course = _get_and_validate_course(course_key_string, request.user)
if (not course and not use_mock_video_uploads()):
return HttpResponseNotFound()
if request.method == "GET":
if "application/json" in request.META.get("HTTP_ACCEPT", ""):
return videos_index_json(course)
pagination_conf = _generate_pagination_configuration(course_key_string, request)
return videos_index_html(course, pagination_conf)
elif request.method == "DELETE":
remove_video_for_course(course_key_string, edx_video_id)
return JsonResponse()
else:
if is_status_update_request(request.json):
return send_video_status_update(request.json)
elif _is_pagination_context_update_request(request):
return _update_pagination_context(request)
data, status = videos_post(course, request)
return JsonResponse(data, status=status)
def handle_generate_video_upload_link(request, course_key_string):
"""
API for creating a video upload. Returns an edx_video_id and a presigned URL that can be used
to upload the video to AWS S3.
"""
course = _get_and_validate_course(course_key_string, request.user)
if not course:
return Response(data='Course Not Found', status=rest_status.HTTP_400_BAD_REQUEST)
data, status = videos_post(course, request)
return Response(data, status=status)
def handle_video_images(request, course_key_string, edx_video_id=None):
"""Function to handle image files"""
# respond with a 404 if image upload is not enabled.
if not VIDEO_IMAGE_UPLOAD_ENABLED.is_enabled() and not use_mock_video_uploads():
return HttpResponseNotFound()
if 'file' not in request.FILES:
return JsonResponse({'error': _('An image file is required.')}, status=400)
image_file = request.FILES['file']
error = validate_video_image(image_file)
if error:
return JsonResponse({'error': error}, status=400)
with closing(image_file):
image_url = update_video_image(edx_video_id, course_key_string, image_file, image_file.name)
LOGGER.info(
'VIDEOS: Video image uploaded for edx_video_id [%s] in course [%s]', edx_video_id, course_key_string
)
return JsonResponse({'image_url': image_url})
def check_video_images_upload_enabled(request):
"""Function to check if images can be uploaded"""
# respond with a false if image upload is not enabled.
if not VIDEO_IMAGE_UPLOAD_ENABLED.is_enabled():
return JsonResponse({'allowThumbnailUpload': False})
return JsonResponse({'allowThumbnailUpload': True})
def enabled_video_features(request):
""" Return a dict with info about which video features are enabled """
features = {
'allowThumbnailUpload': VIDEO_IMAGE_UPLOAD_ENABLED.is_enabled(),
'videoSharingEnabled': PUBLIC_VIDEO_SHARE.is_enabled(),
}
return JsonResponse(features)
def validate_transcript_preferences(provider, cielo24_fidelity, cielo24_turnaround,
three_play_turnaround, video_source_language, preferred_languages):
"""
Validate 3rd Party Transcription Preferences.
Arguments:
provider: Transcription provider
cielo24_fidelity: Cielo24 transcription fidelity.
cielo24_turnaround: Cielo24 transcription turnaround.
three_play_turnaround: 3PlayMedia transcription turnaround.
video_source_language: Source/Speech language of the videos that are going to be submitted to the Providers.
preferred_languages: list of language codes.
Returns:
validated preferences or a validation error.
"""
error, preferences = None, {}
# validate transcription providers
transcription_plans = get_3rd_party_transcription_plans()
if provider in list(transcription_plans.keys()): # lint-amnesty, pylint: disable=consider-iterating-dictionary
# Further validations for providers
if provider == TranscriptProvider.CIELO24:
# Validate transcription fidelity
if cielo24_fidelity in transcription_plans[provider]['fidelity']:
# Validate transcription turnaround
if cielo24_turnaround not in transcription_plans[provider]['turnaround']:
error = f'Invalid cielo24 turnaround {cielo24_turnaround}.'
return error, preferences
# Validate transcription languages
supported_languages = transcription_plans[provider]['fidelity'][cielo24_fidelity]['languages']
if video_source_language not in supported_languages:
error = f'Unsupported source language {video_source_language}.'
return error, preferences
if not preferred_languages or not set(preferred_languages) <= set(supported_languages.keys()):
error = f'Invalid languages {preferred_languages}.'
return error, preferences
# Validated Cielo24 preferences
preferences = {
'video_source_language': video_source_language,
'cielo24_fidelity': cielo24_fidelity,
'cielo24_turnaround': cielo24_turnaround,
'preferred_languages': preferred_languages,
}
else:
error = f'Invalid cielo24 fidelity {cielo24_fidelity}.'
elif provider == TranscriptProvider.THREE_PLAY_MEDIA:
# Validate transcription turnaround
if three_play_turnaround not in transcription_plans[provider]['turnaround']:
error = f'Invalid 3play turnaround {three_play_turnaround}.'
return error, preferences
# Validate transcription languages
valid_translations_map = transcription_plans[provider]['translations']
if video_source_language not in list(valid_translations_map.keys()):
error = f'Unsupported source language {video_source_language}.'
return error, preferences
valid_target_languages = valid_translations_map[video_source_language]
if not preferred_languages or not set(preferred_languages) <= set(valid_target_languages):
error = f'Invalid languages {preferred_languages}.'
return error, preferences
# Validated 3PlayMedia preferences
preferences = {
'three_play_turnaround': three_play_turnaround,
'video_source_language': video_source_language,
'preferred_languages': preferred_languages,
}
else:
error = f'Invalid provider {provider}.'
return error, preferences
def handle_transcript_preferences(request, course_key_string):
"""
JSON view handler to post the transcript preferences.
Arguments:
request: WSGI request object
course_key_string: string for course key
Returns: valid json response or 400 with error message
"""
course_key = CourseKey.from_string(course_key_string)
is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course_key)
if not is_video_transcript_enabled:
return HttpResponseNotFound()
if request.method == 'POST':
data = request.json
provider = data.get('provider')
error, preferences = validate_transcript_preferences(
provider=provider,
cielo24_fidelity=data.get('cielo24_fidelity', ''),
cielo24_turnaround=data.get('cielo24_turnaround', ''),
three_play_turnaround=data.get('three_play_turnaround', ''),
video_source_language=data.get('video_source_language'),
preferred_languages=list(map(str, data.get('preferred_languages', [])))
)
if error:
response = JsonResponse({'error': error}, status=400)
else:
preferences.update({'provider': provider})
transcript_preferences = create_or_update_transcript_preferences(course_key_string, **preferences)
response = JsonResponse({'transcript_preferences': transcript_preferences}, status=200)
return response
elif request.method == 'DELETE':
remove_transcript_preferences(course_key_string)
return JsonResponse()
def get_video_encodings_download(request, course_key_string):
"""
Returns a CSV report containing the encoded video URLs for video uploads
in the following format:
Video ID,Name,Status,Profile1 URL,Profile2 URL
aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa,video.mp4,Complete,http://example.com/prof1.mp4,http://example.com/prof2.mp4
"""
course = _get_and_validate_course(course_key_string, request.user)
if not course:
return HttpResponseNotFound()
def get_profile_header(profile):
"""Returns the column header string for the given profile's URLs"""
# Translators: This is the header for a CSV file column
# containing URLs for video encodings for the named profile
# (e.g. desktop, mobile high quality, mobile low quality)
return _("{profile_name} URL").format(profile_name=profile)
profile_whitelist = VideoUploadConfig.get_profile_whitelist()
videos, __ = _get_videos(course)
videos = list(videos)
name_col = _("Name")
duration_col = _("Duration")
added_col = _("Date Added")
video_id_col = _("Video ID")
status_col = _("Status")
profile_cols = [get_profile_header(profile) for profile in profile_whitelist]
def make_csv_dict(video):
"""
Makes a dictionary suitable for writing CSV output. This involves
extracting the required items from the original video dict and
converting all keys and values to UTF-8 encoded string objects,
because the CSV module doesn't play well with unicode objects.
"""
# Translators: This is listed as the duration for a video that has not
# yet reached the point in its processing by the servers where its
# duration is determined.
duration_val = str(video["duration"]) if video["duration"] > 0 else _("Pending")
ret = dict(
[
(name_col, video["client_video_id"]),
(duration_col, duration_val),
(added_col, video["created"].isoformat()),
(video_id_col, video["edx_video_id"]),
(status_col, video["status"]),
] +
[
(get_profile_header(encoded_video["profile"]), encoded_video["url"])
for encoded_video in video["encoded_videos"]
if encoded_video["profile"] in profile_whitelist
]
)
return dict(ret.items())
# Write csv to bytes-like object. We need a separate writer and buffer as the csv
# writer writes str and the FileResponse expects a bytes files.
buffer = io.BytesIO()
buffer_writer = codecs.getwriter("utf-8")(buffer)
writer = csv.DictWriter(
buffer_writer,
[name_col, duration_col, added_col, video_id_col, status_col] + profile_cols,
dialect=csv.excel
)
writer.writeheader()
for video in videos:
writer.writerow(make_csv_dict(video))
buffer.seek(0)
# Translators: This is the suggested filename when downloading the URL
# listing for videos uploaded through Studio
filename = _("{course}_video_urls").format(course=course.id.course) + ".csv"
return FileResponse(buffer, as_attachment=True, filename=filename, content_type="text/csv")
def _get_and_validate_course(course_key_string, user):
"""
Given a course key, return the course if it exists, the given user has
access to it, and it is properly configured for video uploads
"""
course_key = CourseKey.from_string(course_key_string)
# For now, assume all studio users that have access to the course can upload videos.
# In the future, we plan to add a new org-level role for video uploaders.
course = get_course_and_check_access(course_key, user)
if (
settings.FEATURES["ENABLE_VIDEO_UPLOAD_PIPELINE"] and
getattr(settings, "VIDEO_UPLOAD_PIPELINE", None) and
course and
course.video_pipeline_configured
):
return course
else:
return None
def convert_video_status(video, is_video_encodes_ready=False):
"""
Convert status of a video. Status can be converted to one of the following:
* FAILED if video is in `upload` state for more than 24 hours
* `YouTube Duplicate` if status is `invalid_token`
* user-friendly video status
"""
now = datetime.now(video.get('created', datetime.now().replace(tzinfo=UTC)).tzinfo)
if video['status'] == 'upload' and (now - video['created']) > timedelta(hours=MAX_UPLOAD_HOURS):
new_status = 'upload_failed'
status = StatusDisplayStrings.get(new_status)
message = 'Video with id [{}] is still in upload after [{}] hours, setting status to [{}]'.format(
video['edx_video_id'], MAX_UPLOAD_HOURS, new_status
)
send_video_status_update([
{
'edxVideoId': video['edx_video_id'],
'status': new_status,
'message': message
}
])
elif video['status'] == 'invalid_token':
status = StatusDisplayStrings.get('youtube_duplicate')
elif is_video_encodes_ready:
status = StatusDisplayStrings.get('file_complete')
else:
status = StatusDisplayStrings.get(video['status'])
return status
def _get_videos(course, pagination_conf=None):
"""
Retrieves the list of videos from VAL corresponding to this course.
"""
videos, pagination_context = get_videos_for_course(
str(course.id),
VideoSortField.created,
SortDirection.desc,
pagination_conf
)
videos = list(videos)
# This is required to see if edx video pipeline is enabled while converting the video status.
course_video_upload_token = course.video_upload_pipeline.get('course_video_upload_token')
transcription_statuses = ['transcription_in_progress', 'transcript_ready', 'partial_failure', 'transcript_failed']
# convert VAL's status to studio's Video Upload feature status.
for video in videos:
# If we are using "new video workflow" and status is in `transcription_statuses` then video encodes are ready.
# This is because Transcription starts once all the encodes are complete except for YT, but according to
# "new video workflow" YT is disabled as well as deprecated. So, Its precise to say that the Transcription
# starts once all the encodings are complete *for the new video workflow*.
is_video_encodes_ready = not course_video_upload_token and (video['status'] in transcription_statuses)
# Update with transcript languages
video['transcripts'] = get_available_transcript_languages(video_id=video['edx_video_id'])
video['transcription_status'] = (
StatusDisplayStrings.get(video['status']) if is_video_encodes_ready else ''
)
video['transcript_urls'] = {}
for language_code in video['transcripts']:
video['transcript_urls'][language_code] = get_video_transcript_url(
video_id=video['edx_video_id'],
language_code=language_code,
)
# Convert the video status.
video['status'] = convert_video_status(video, is_video_encodes_ready)
return videos, pagination_context
def _get_default_video_image_url():
"""
Returns default video image url
"""
return staticfiles_storage.url(settings.VIDEO_IMAGE_DEFAULT_FILENAME)
def _get_index_videos(course, pagination_conf=None):
"""
Returns the information about each video upload required for the video list
"""
course_id = str(course.id)
attrs = [
'edx_video_id', 'client_video_id', 'created', 'duration',
'status', 'courses', 'transcripts', 'transcription_status',
'transcript_urls', 'error_description'
]
def _get_values(video):
"""
Get data for predefined video attributes.
"""
values = {}
for attr in attrs:
if attr == 'courses':
course = [c for c in video['courses'] if course_id in c]
(__, values['course_video_image_url']), = list(course[0].items())
else:
values[attr] = video[attr]
return values
videos, pagination_context = _get_videos(course, pagination_conf)
return [_get_values(video) for video in videos], pagination_context
def get_all_transcript_languages():
"""
Returns all possible languages for transcript.
"""
third_party_transcription_languages = {}
transcription_plans = get_3rd_party_transcription_plans()
cielo_fidelity = transcription_plans[TranscriptProvider.CIELO24]['fidelity']
# Get third party transcription languages.
third_party_transcription_languages.update(transcription_plans[TranscriptProvider.THREE_PLAY_MEDIA]['languages'])
third_party_transcription_languages.update(cielo_fidelity['MECHANICAL']['languages'])
third_party_transcription_languages.update(cielo_fidelity['PREMIUM']['languages'])
third_party_transcription_languages.update(cielo_fidelity['PROFESSIONAL']['languages'])
all_languages_dict = dict(settings.ALL_LANGUAGES, **third_party_transcription_languages)
# Return combined system settings and 3rd party transcript languages.
all_languages = []
for key, value in sorted(all_languages_dict.items(), key=lambda k_v: k_v[1]):
all_languages.append({
'language_code': key,
'language_text': value
})
return all_languages
def videos_index_html(course, pagination_conf=None):
"""
Returns an HTML page to display previous video uploads and allow new ones
"""
is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course.id)
previous_uploads, pagination_context = _get_index_videos(course, pagination_conf)
context = {
'context_course': course,
'image_upload_url': reverse_course_url('video_images_handler', str(course.id)),
'video_handler_url': reverse_course_url('videos_handler', str(course.id)),
'encodings_download_url': reverse_course_url('video_encodings_download', str(course.id)),
'default_video_image_url': _get_default_video_image_url(),
'previous_uploads': previous_uploads,
'concurrent_upload_limit': settings.VIDEO_UPLOAD_PIPELINE.get('CONCURRENT_UPLOAD_LIMIT', 0),
'video_supported_file_formats': list(VIDEO_SUPPORTED_FILE_FORMATS.keys()),
'video_upload_max_file_size': VIDEO_UPLOAD_MAX_FILE_SIZE_GB,
'video_image_settings': {
'video_image_upload_enabled': VIDEO_IMAGE_UPLOAD_ENABLED.is_enabled(),
'max_size': settings.VIDEO_IMAGE_SETTINGS['VIDEO_IMAGE_MAX_BYTES'],
'min_size': settings.VIDEO_IMAGE_SETTINGS['VIDEO_IMAGE_MIN_BYTES'],
'max_width': settings.VIDEO_IMAGE_MAX_WIDTH,
'max_height': settings.VIDEO_IMAGE_MAX_HEIGHT,
'supported_file_formats': settings.VIDEO_IMAGE_SUPPORTED_FILE_FORMATS
},
'is_video_transcript_enabled': is_video_transcript_enabled,
'active_transcript_preferences': None,
'transcript_credentials': None,
'transcript_available_languages': get_all_transcript_languages(),
'video_transcript_settings': {
'transcript_download_handler_url': reverse('transcript_download_handler'),
'transcript_upload_handler_url': reverse('transcript_upload_handler'),
'transcript_delete_handler_url': reverse_course_url('transcript_delete_handler', str(course.id)),
'trancript_download_file_format': Transcript.SRT
},
'pagination_context': pagination_context
}
if is_video_transcript_enabled:
context['video_transcript_settings'].update({
'transcript_preferences_handler_url': reverse_course_url(
'transcript_preferences_handler',
str(course.id)
),
'transcript_credentials_handler_url': reverse_course_url(
'transcript_credentials_handler',
str(course.id)
),
'transcription_plans': get_3rd_party_transcription_plans(),
})
context['active_transcript_preferences'] = get_transcript_preferences(str(course.id))
# Cached state for transcript providers' credentials (org-specific)
context['transcript_credentials'] = get_transcript_credentials_state_for_org(course.id.org)
if use_new_video_uploads_page(course.id):
return redirect(get_video_uploads_url(course.id))
return render_to_response('videos_index.html', context)
def videos_index_json(course):
"""
Returns JSON in the following format:
{
'videos': [{
'edx_video_id': 'aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa',
'client_video_id': 'video.mp4',
'created': '1970-01-01T00:00:00Z',
'duration': 42.5,
'status': 'upload',
'course_video_image_url': 'https://video/images/1234.jpg'
}]
}
"""
index_videos, __ = _get_index_videos(course)
return JsonResponse({"videos": index_videos}, status=200)
def videos_post(course, request):
"""
Input (JSON):
{
"files": [{
"file_name": "video.mp4",
"content_type": "video/mp4"
}]
}
Returns (JSON):
{
"files": [{
"file_name": "video.mp4",
"upload_url": "http://example.com/put_video"
}]
}
The returned array corresponds exactly to the input array.
"""
if use_mock_video_uploads():
return {'files': [{'file_name': 'video.mp4', 'upload_url': 'http://example.com/put_video'}]}, 200
error = None
data = request.json
if 'files' not in data:
error = "Request object is not JSON or does not contain 'files'"
elif any(
'file_name' not in file or 'content_type' not in file
for file in data['files']
):
error = "Request 'files' entry does not contain 'file_name' and 'content_type'"
elif any(
file['content_type'] not in list(VIDEO_SUPPORTED_FILE_FORMATS.values())
for file in data['files']
):
error = "Request 'files' entry contain unsupported content_type"
if error:
return {'error': error}, 400
bucket = storage_service_bucket()
req_files = data['files']
resp_files = []
for req_file in req_files:
file_name = req_file['file_name']
try:
file_name.encode('ascii')
except UnicodeEncodeError:
error_msg = 'The file name for %s must contain only ASCII characters.' % file_name
return {'error': error_msg}, 400
edx_video_id = str(uuid4())
key = storage_service_key(bucket, file_name=edx_video_id)
metadata_list = [
('client_video_id', file_name),
('course_key', str(course.id)),
]
course_video_upload_token = course.video_upload_pipeline.get('course_video_upload_token')
# Only include `course_video_upload_token` if youtube has not been deprecated
# for this course.
if not DEPRECATE_YOUTUBE.is_enabled(course.id) and course_video_upload_token:
metadata_list.append(('course_video_upload_token', course_video_upload_token))
is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course.id)
if is_video_transcript_enabled:
transcript_preferences = get_transcript_preferences(str(course.id))
if transcript_preferences is not None:
metadata_list.append(('transcript_preferences', json.dumps(transcript_preferences)))
for metadata_name, value in metadata_list:
key.set_metadata(metadata_name, value)
upload_url = key.generate_url(
KEY_EXPIRATION_IN_SECONDS,
'PUT',
headers={'Content-Type': req_file['content_type']}
)
# persist edx_video_id in VAL
create_video({
'edx_video_id': edx_video_id,
'status': 'upload',
'client_video_id': file_name,
'duration': 0,
'encoded_videos': [],
'courses': [str(course.id)]
})
resp_files.append({'file_name': file_name, 'upload_url': upload_url, 'edx_video_id': edx_video_id})
return {'files': resp_files}, 200
def storage_service_bucket():
"""
Returns an S3 bucket for video upload.
"""
if ENABLE_DEVSTACK_VIDEO_UPLOADS.is_enabled():
params = {
'aws_access_key_id': settings.AWS_ACCESS_KEY_ID,
'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY,
'security_token': settings.AWS_SECURITY_TOKEN
}
else:
params = {
'aws_access_key_id': settings.AWS_ACCESS_KEY_ID,
'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY
}
conn = S3Connection(**params)
# We don't need to validate our bucket, it requires a very permissive IAM permission
# set since behind the scenes it fires a HEAD request that is equivalent to get_all_keys()
# meaning it would need ListObjects on the whole bucket, not just the path used in each
# environment (since we share a single bucket for multiple deployments in some configurations)
return conn.get_bucket(settings.VIDEO_UPLOAD_PIPELINE['VEM_S3_BUCKET'], validate=False)
def storage_service_key(bucket, file_name):
"""
Returns an S3 key to the given file in the given bucket.
"""
key_name = "{}/{}".format(
settings.VIDEO_UPLOAD_PIPELINE.get("ROOT_PATH", ""),
file_name
)
return s3.key.Key(bucket, key_name)
def send_video_status_update(updates):
"""
Update video status in edx-val.
"""
for update in updates:
update_video_status(update.get('edxVideoId'), update.get('status'))
LOGGER.info(
'VIDEOS: Video status update with id [%s], status [%s] and message [%s]',
update.get('edxVideoId'),
update.get('status'),
update.get('message')
)
return JsonResponse()
def is_status_update_request(request_data):
"""
Returns True if `request_data` contains status update else False.
"""
return any('status' in update for update in request_data)
def _generate_pagination_configuration(course_key_string, request):
"""
Returns pagination configuration
"""
course_key = CourseKey.from_string(course_key_string)
if not ENABLE_VIDEO_UPLOAD_PAGINATION.is_enabled(course_key):
return None
return {
'page_number': request.GET.get('page', 1),
'videos_per_page': request.session.get("VIDEOS_PER_PAGE", VIDEOS_PER_PAGE)
}
def _is_pagination_context_update_request(request):
"""
Checks if request contains `videos_per_page`
"""
return request.POST.get('id', '') == "videos_per_page"
def _update_pagination_context(request):
"""
Updates session with posted value
"""
error_msg = _('A non zero positive integer is expected')
try:
videos_per_page = int(request.POST.get('value'))
if videos_per_page <= 0:
return JsonResponse({'error': error_msg}, status=500)
except ValueError:
return JsonResponse({'error': error_msg}, status=500)
request.session['VIDEOS_PER_PAGE'] = videos_per_page
return JsonResponse()

34
cms/djangoapps/contentstore/views/tests/test_transcript_settings.py
View File

@@ -12,13 +12,15 @@ from django.urls import reverse
from edxval import api
from cms.djangoapps.contentstore.tests.utils import CourseTestCase
from cms.djangoapps.contentstore.transcript_storage_handlers import (
TranscriptionProviderErrorType,
validate_transcript_credentials
)
from cms.djangoapps.contentstore.utils import reverse_course_url
from common.djangoapps.student.roles import CourseStaffRole
from openedx.core.djangoapps.profile_images.tests.helpers import make_image_file
from openedx.core.djangoapps.oauth_dispatch.jwt import create_jwt_for_user
from ..transcript_settings import TranscriptionProviderErrorType, validate_transcript_credentials
@ddt.ddt
@patch(
@@ -94,7 +96,7 @@ class TranscriptCredentialsTest(CourseTestCase):
)
)
@ddt.unpack
@patch('cms.djangoapps.contentstore.views.transcript_settings.update_3rd_party_transcription_service_credentials')
@patch('cms.djangoapps.contentstore.transcript_storage_handlers.update_3rd_party_transcription_service_credentials')
def test_transcript_credentials_handler(self, request_payload, update_credentials_response, expected_status_code,
expected_response, mock_update_credentials):
"""
@@ -211,7 +213,7 @@ class TranscriptDownloadTest(CourseTestCase):
response = self.client.post(self.view_url, content_type='application/json')
self.assertEqual(response.status_code, 405)
@patch('cms.djangoapps.contentstore.views.transcript_settings.get_video_transcript_data')
@patch('cms.djangoapps.contentstore.transcript_storage_handlers.get_video_transcript_data')
def test_transcript_download_handler(self, mock_get_video_transcript_data):
"""
Tests that transcript download handler works as expected.
@@ -303,9 +305,9 @@ class TranscriptUploadTest(CourseTestCase):
response = self.client.get(self.view_url, content_type='application/json')
self.assertEqual(response.status_code, 405)
@patch('cms.djangoapps.contentstore.views.transcript_settings.create_or_update_video_transcript')
@patch('cms.djangoapps.contentstore.transcript_storage_handlers.create_or_update_video_transcript')
@patch(
'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages',
'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages',
Mock(return_value=['en']),
)
def test_transcript_upload_handler(self, mock_create_or_update_video_transcript):
@@ -370,7 +372,7 @@ class TranscriptUploadTest(CourseTestCase):
)
@ddt.unpack
@patch(
'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages',
'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages',
Mock(return_value=['en']),
)
def test_transcript_upload_handler_missing_attrs(self, request_payload, expected_error_message):
@@ -383,7 +385,7 @@ class TranscriptUploadTest(CourseTestCase):
self.assertEqual(json.loads(response.content.decode('utf-8'))['error'], expected_error_message)
@patch(
'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages',
'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages',
Mock(return_value=['en', 'es'])
)
def test_transcript_upload_handler_existing_transcript(self):
@@ -405,7 +407,7 @@ class TranscriptUploadTest(CourseTestCase):
)
@patch(
'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages',
'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages',
Mock(return_value=['en']),
)
def test_transcript_upload_handler_with_image(self):
@@ -432,7 +434,7 @@ class TranscriptUploadTest(CourseTestCase):
)
@patch(
'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages',
'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages',
Mock(return_value=['en']),
)
def test_transcript_upload_handler_with_invalid_transcript(self):
@@ -588,9 +590,9 @@ class TranscriptUploadApiTest(CourseTestCase):
response = self.client.get(self.view_url, content_type='application/json')
self.assertEqual(response.status_code, 405)
@patch('cms.djangoapps.contentstore.views.transcript_settings.create_or_update_video_transcript')
@patch('cms.djangoapps.contentstore.transcript_storage_handlers.create_or_update_video_transcript')
@patch(
'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages',
'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages',
Mock(return_value=['en']),
)
def test_transcript_upload_handler(self, mock_create_or_update_video_transcript):
@@ -655,7 +657,7 @@ class TranscriptUploadApiTest(CourseTestCase):
)
@ddt.unpack
@patch(
'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages',
'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages',
Mock(return_value=['en']),
)
def test_transcript_upload_handler_missing_attrs(self, request_payload, expected_error_message):
@@ -668,7 +670,7 @@ class TranscriptUploadApiTest(CourseTestCase):
self.assertEqual(json.loads(response.content.decode('utf-8'))['error'], expected_error_message)
@patch(
'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages',
'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages',
Mock(return_value=['en', 'es'])
)
def test_transcript_upload_handler_existing_transcript(self):
@@ -690,7 +692,7 @@ class TranscriptUploadApiTest(CourseTestCase):
)
@patch(
'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages',
'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages',
Mock(return_value=['en']),
)
def test_transcript_upload_handler_with_image(self):
@@ -717,7 +719,7 @@ class TranscriptUploadApiTest(CourseTestCase):
)
@patch(
'cms.djangoapps.contentstore.views.transcript_settings.get_available_transcript_languages',
'cms.djangoapps.contentstore.transcript_storage_handlers.get_available_transcript_languages',
Mock(return_value=['en']),
)
def test_transcript_upload_handler_with_invalid_transcript(self):

30
cms/djangoapps/contentstore/views/tests/test_videos.py
View File

@@ -43,11 +43,15 @@ from ..videos import (
ENABLE_VIDEO_UPLOAD_PAGINATION,
KEY_EXPIRATION_IN_SECONDS,
VIDEO_IMAGE_UPLOAD_ENABLED,
PUBLIC_VIDEO_SHARE,
StatusDisplayStrings,
TranscriptProvider,
)
from cms.djangoapps.contentstore.video_storage_handlers import (
_get_default_video_image_url,
convert_video_status, storage_service_bucket, storage_service_key
TranscriptProvider,
StatusDisplayStrings,
convert_video_status,
storage_service_bucket,
storage_service_key,
PUBLIC_VIDEO_SHARE
)
@@ -210,7 +214,7 @@ class VideoUploadPostTestsMixin:
"""
@override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret')
@patch('boto.s3.key.Key')
@patch('cms.djangoapps.contentstore.views.videos.S3Connection')
@patch('cms.djangoapps.contentstore.video_storage_handlers.S3Connection')
def test_post_success(self, mock_conn, mock_key):
files = [
{
@@ -467,7 +471,7 @@ class VideosHandlerTestCase(
@override_settings(AWS_ACCESS_KEY_ID="test_key_id", AWS_SECRET_ACCESS_KEY="test_secret")
@patch("boto.s3.key.Key")
@patch("cms.djangoapps.contentstore.views.videos.S3Connection")
@patch("cms.djangoapps.contentstore.video_storage_handlers.S3Connection")
@ddt.data(
(
[
@@ -529,7 +533,7 @@ class VideosHandlerTestCase(
self.assertEqual(response['error'], "Request 'files' entry contain unsupported content_type")
@override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret')
@patch('cms.djangoapps.contentstore.views.videos.S3Connection')
@patch('cms.djangoapps.contentstore.video_storage_handlers.S3Connection')
def test_upload_with_non_ascii_charaters(self, mock_conn):
"""
Test that video uploads throws error message when file name contains special characters.
@@ -552,7 +556,7 @@ class VideosHandlerTestCase(
@override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret', AWS_SECURITY_TOKEN='token')
@patch('boto.s3.key.Key')
@patch('cms.djangoapps.contentstore.views.videos.S3Connection')
@patch('cms.djangoapps.contentstore.video_storage_handlers.S3Connection')
@override_waffle_flag(ENABLE_DEVSTACK_VIDEO_UPLOADS, active=True)
def test_devstack_upload_connection(self, mock_conn, mock_key):
files = [{'file_name': 'first.mp4', 'content_type': 'video/mp4'}]
@@ -580,7 +584,7 @@ class VideosHandlerTestCase(
)
@patch('boto.s3.key.Key')
@patch('cms.djangoapps.contentstore.views.videos.S3Connection')
@patch('cms.djangoapps.contentstore.video_storage_handlers.S3Connection')
def test_send_course_to_vem_pipeline(self, mock_conn, mock_key):
"""
Test that uploads always go to VEM S3 bucket by default.
@@ -610,7 +614,7 @@ class VideosHandlerTestCase(
@override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret')
@patch('boto.s3.key.Key')
@patch('cms.djangoapps.contentstore.views.videos.S3Connection')
@patch('cms.djangoapps.contentstore.video_storage_handlers.S3Connection')
@ddt.data(
{
'global_waffle': True,
@@ -770,7 +774,7 @@ class VideosHandlerTestCase(
# Test should fail if video not found
self.assertEqual(True, False, 'Invalid edx_video_id')
@patch('cms.djangoapps.contentstore.views.videos.LOGGER')
@patch('cms.djangoapps.contentstore.video_storage_handlers.LOGGER')
def test_video_status_update_request(self, mock_logger):
"""
Verifies that video status update request works as expected.
@@ -1447,8 +1451,8 @@ class TranscriptPreferencesTestCase(VideoUploadTestBase, CourseTestCase):
@ddt.unpack
@override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret')
@patch('boto.s3.key.Key')
@patch('cms.djangoapps.contentstore.views.videos.S3Connection')
@patch('cms.djangoapps.contentstore.views.videos.get_transcript_preferences')
@patch('cms.djangoapps.contentstore.video_storage_handlers.S3Connection')
@patch('cms.djangoapps.contentstore.video_storage_handlers.get_transcript_preferences')
def test_transcript_preferences_metadata(self, transcript_preferences, is_video_transcript_enabled,
mock_transcript_preferences, mock_conn, mock_key):
"""

203
cms/djangoapps/contentstore/views/transcript_settings.py
View File

@@ -4,32 +4,23 @@ Views related to the transcript preferences feature
import logging
import os
from django.contrib.auth.decorators import login_required
from django.core.files.base import ContentFile
from django.http import HttpResponse, HttpResponseNotFound
from django.utils.translation import gettext as _
from django.http import HttpResponseNotFound
from django.views.decorators.http import require_GET, require_http_methods, require_POST
from edxval.api import (
create_or_update_video_transcript,
delete_video_transcript,
get_3rd_party_transcription_plans,
get_available_transcript_languages,
get_video_transcript_data,
update_transcript_credentials_state_for_org
)
from opaque_keys.edx.keys import CourseKey
from rest_framework.decorators import api_view
from cms.djangoapps.contentstore.transcript_storage_handlers import (
validate_transcript_upload_data,
upload_transcript,
delete_video_transcript,
handle_transcript_credentials,
handle_transcript_download,
)
from common.djangoapps.student.auth import has_studio_write_access
from common.djangoapps.util.json_request import JsonResponse, expect_json
from openedx.core.djangoapps.video_config.models import VideoTranscriptEnabledFlag
from openedx.core.djangoapps.video_pipeline.api import update_3rd_party_transcription_service_credentials
from openedx.core.lib.api.view_utils import view_auth_classes
from xmodule.video_block.transcripts_utils import Transcript, TranscriptsGenerationException # lint-amnesty, pylint: disable=wrong-import-order
from .videos import TranscriptProvider
__all__ = [
'transcript_credentials_handler',
@@ -42,51 +33,6 @@ __all__ = [
LOGGER = logging.getLogger(__name__)
class TranscriptionProviderErrorType:
"""
Transcription provider's error types enumeration.
"""
INVALID_CREDENTIALS = 1
def validate_transcript_credentials(provider, **credentials):
"""
Validates transcript credentials.
Validations:
Providers must be either 3PlayMedia or Cielo24.
In case of:
3PlayMedia - 'api_key' and 'api_secret_key' are required.
Cielo24 - 'api_key' and 'username' are required.
It ignores any extra/unrelated parameters passed in credentials and
only returns the validated ones.
"""
error_message, validated_credentials = '', {}
valid_providers = list(get_3rd_party_transcription_plans().keys())
if provider in valid_providers:
must_have_props = []
if provider == TranscriptProvider.THREE_PLAY_MEDIA:
must_have_props = ['api_key', 'api_secret_key']
elif provider == TranscriptProvider.CIELO24:
must_have_props = ['api_key', 'username']
missing = [
must_have_prop for must_have_prop in must_have_props if must_have_prop not in list(credentials.keys()) # lint-amnesty, pylint: disable=consider-iterating-dictionary
]
if missing:
error_message = '{missing} must be specified.'.format(missing=' and '.join(missing))
return error_message, validated_credentials
validated_credentials.update({
prop: credentials[prop] for prop in must_have_props
})
else:
error_message = f'Invalid Provider {provider}.'
return error_message, validated_credentials
@expect_json
@login_required
@require_POST
@@ -103,35 +49,7 @@ def transcript_credentials_handler(request, course_key_string):
- A 404 response if transcript feature is not enabled for this course.
- A 400 if credentials do not pass validations, hence not updated in edx-video-pipeline.
"""
course_key = CourseKey.from_string(course_key_string)
if not VideoTranscriptEnabledFlag.feature_enabled(course_key):
return HttpResponseNotFound()
provider = request.json.pop('provider')
error_message, validated_credentials = validate_transcript_credentials(provider=provider, **request.json)
if error_message:
response = JsonResponse({'error': error_message}, status=400)
else:
# Send the validated credentials to edx-video-pipeline and video-encode-manager
credentials_payload = dict(validated_credentials, org=course_key.org, provider=provider)
error_response, is_updated = update_3rd_party_transcription_service_credentials(**credentials_payload)
# Send appropriate response based on whether credentials were updated or not.
if is_updated:
# Cache credentials state in edx-val.
update_transcript_credentials_state_for_org(org=course_key.org, provider=provider, exists=is_updated)
response = JsonResponse(status=200)
else:
# Error response would contain error types and the following
# error type is received from edx-video-pipeline whenever we've
# got invalid credentials for a provider. Its kept this way because
# edx-video-pipeline doesn't support i18n translations yet.
error_type = error_response.get('error_type')
if error_type == TranscriptionProviderErrorType.INVALID_CREDENTIALS:
error_message = _('The information you entered is incorrect.')
response = JsonResponse({'error': error_message}, status=400)
return response
return handle_transcript_credentials(request, course_key_string)
@login_required
@@ -148,112 +66,17 @@ def transcript_download_handler(request):
- A 400 if there is a validation error.
- A 404 if there is no such transcript.
"""
missing = [attr for attr in ['edx_video_id', 'language_code'] if attr not in request.GET]
if missing:
return JsonResponse(
{'error': _('The following parameters are required: {missing}.').format(missing=', '.join(missing))},
status=400
)
edx_video_id = request.GET['edx_video_id']
language_code = request.GET['language_code']
transcript = get_video_transcript_data(video_id=edx_video_id, language_code=language_code)
if transcript:
name_and_extension = os.path.splitext(transcript['file_name'])
basename, file_format = name_and_extension[0], name_and_extension[1][1:]
transcript_filename = f'{basename}.{Transcript.SRT}'
transcript_content = Transcript.convert(
content=transcript['content'],
input_format=file_format,
output_format=Transcript.SRT
)
# Construct an HTTP response
response = HttpResponse(transcript_content, content_type=Transcript.mime_types[Transcript.SRT])
response['Content-Disposition'] = f'attachment; filename="{transcript_filename}"'
else:
response = HttpResponseNotFound()
return response
def upload_transcript(request):
"""
Upload a transcript file
Arguments:
request: A WSGI request object
Transcript file in SRT format
"""
edx_video_id = request.POST['edx_video_id']
language_code = request.POST['language_code']
new_language_code = request.POST['new_language_code']
transcript_file = request.FILES['file']
try:
# Convert SRT transcript into an SJSON format
# and upload it to S3.
sjson_subs = Transcript.convert(
content=transcript_file.read().decode('utf-8'),
input_format=Transcript.SRT,
output_format=Transcript.SJSON
).encode()
create_or_update_video_transcript(
video_id=edx_video_id,
language_code=language_code,
metadata={
'provider': TranscriptProvider.CUSTOM,
'file_format': Transcript.SJSON,
'language_code': new_language_code
},
file_data=ContentFile(sjson_subs),
)
response = JsonResponse(status=201)
except (TranscriptsGenerationException, UnicodeDecodeError):
LOGGER.error("Unable to update transcript on edX video %s for language %s", edx_video_id, new_language_code)
response = JsonResponse(
{'error': _('There is a problem with this transcript file. Try to upload a different file.')},
status=400
)
finally:
LOGGER.info("Updated transcript on edX video %s for language %s", edx_video_id, new_language_code)
return response
def validate_transcript_upload_data(data, files):
"""
Validates video transcript file.
Arguments:
data: A request's data part.
files: A request's files part.
Returns:
None or String
If there is error returns error message otherwise None.
"""
error = None
# Validate the must have attributes - this error is unlikely to be faced by common users.
must_have_attrs = ['edx_video_id', 'language_code', 'new_language_code']
missing = [attr for attr in must_have_attrs if attr not in data]
if missing:
error = _('The following parameters are required: {missing}.').format(missing=', '.join(missing))
elif (
data['language_code'] != data['new_language_code'] and
data['new_language_code'] in get_available_transcript_languages(video_id=data['edx_video_id'])
):
error = _('A transcript with the "{language_code}" language code already exists.'.format( # lint-amnesty, pylint: disable=translation-of-non-string
language_code=data['new_language_code']
))
elif 'file' not in files:
error = _('A transcript file is required.')
return error
return handle_transcript_download(request)
# New version of this transcript upload API in contentstore/rest_api/transcripts.py
# Keeping the old API for backward compatibility
@api_view(['POST'])
@view_auth_classes()
@expect_json
def transcript_upload_api(request):
"""
API View for uploading transcript files.
(Old) API View for uploading transcript files.
Arguments:
request: A WSGI request object

2
cms/djangoapps/contentstore/views/transcripts_ajax.py
View File

@@ -21,7 +21,7 @@ from edxval.api import create_external_video, create_or_update_video_transcript
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import UsageKey
from cms.djangoapps.contentstore.views.videos import TranscriptProvider
from cms.djangoapps.contentstore.video_storage_handlers import TranscriptProvider
from common.djangoapps.student.auth import has_course_author_access
from common.djangoapps.util.json_request import JsonResponse
from xmodule.contentstore.content import StaticContent # lint-amnesty, pylint: disable=wrong-import-order

790
cms/djangoapps/contentstore/views/videos.py
View File

@@ -3,65 +3,35 @@ Views related to the video upload feature
"""
import codecs
import csv
import io
import json
import logging
from contextlib import closing
from datetime import datetime, timedelta
from uuid import uuid4
from boto.s3.connection import S3Connection
from boto import s3
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.contrib.staticfiles.storage import staticfiles_storage
from django.http import FileResponse, HttpResponseNotFound
from django.shortcuts import redirect
from django.urls import reverse
from django.utils.translation import gettext as _
from django.utils.translation import gettext_noop
from django.views.decorators.http import require_GET, require_http_methods, require_POST
from edx_toggles.toggles import WaffleSwitch
from edxval.api import (
SortDirection,
VideoSortField,
create_or_update_transcript_preferences,
create_video,
get_3rd_party_transcription_plans,
get_available_transcript_languages,
get_video_transcript_url,
get_transcript_credentials_state_for_org,
get_transcript_preferences,
get_videos_for_course,
remove_transcript_preferences,
remove_video_for_course,
update_video_image,
update_video_status
)
from opaque_keys.edx.keys import CourseKey
from pytz import UTC
from rest_framework import status as rest_status
from rest_framework.decorators import api_view
from rest_framework.response import Response
from common.djangoapps.edxmako.shortcuts import render_to_response
from common.djangoapps.util.json_request import JsonResponse, expect_json
from openedx.core.djangoapps.video_config.models import VideoTranscriptEnabledFlag
from openedx.core.djangoapps.video_config.toggles import PUBLIC_VIDEO_SHARE
from openedx.core.djangoapps.video_pipeline.config.waffle import (
DEPRECATE_YOUTUBE,
ENABLE_DEVSTACK_VIDEO_UPLOADS,
from cms.djangoapps.contentstore.video_storage_handlers import (
handle_videos,
handle_generate_video_upload_link,
handle_video_images,
check_video_images_upload_enabled,
enabled_video_features,
handle_transcript_preferences,
get_video_encodings_download,
validate_transcript_preferences as validate_transcript_preferences_source_function,
convert_video_status as convert_video_status_source_function,
get_all_transcript_languages as get_all_transcript_languages_source_function,
videos_index_html as videos_index_html_source_function,
videos_index_json as videos_index_json_source_function,
videos_post as videos_post_source_function,
storage_service_bucket as storage_service_bucket_source_function,
storage_service_key as storage_service_key_source_function,
send_video_status_update as send_video_status_update_source_function,
is_status_update_request as is_status_update_request_source_function,
)
from common.djangoapps.util.json_request import expect_json
from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag
from openedx.core.lib.api.view_utils import view_auth_classes
from xmodule.video_block.transcripts_utils import Transcript # lint-amnesty, pylint: disable=wrong-import-order
from ..models import VideoUploadConfig
from ..toggles import use_new_video_uploads_page
from ..utils import reverse_course_url, get_video_uploads_url
from ..video_utils import validate_video_image
from .course import get_course_and_check_access
__all__ = [
'videos_handler',
@@ -105,85 +75,6 @@ MAX_UPLOAD_HOURS = 24
VIDEOS_PER_PAGE = 100
class TranscriptProvider:
"""
Transcription Provider Enumeration
"""
CIELO24 = 'Cielo24'
THREE_PLAY_MEDIA = '3PlayMedia'
CUSTOM = 'Custom'
class StatusDisplayStrings:
"""
A class to map status strings as stored in VAL to display strings for the
video upload page
"""
# Translators: This is the status of an active video upload
_UPLOADING = gettext_noop("Uploading")
# Translators: This is the status for a video that the servers are currently processing
_IN_PROGRESS = gettext_noop("In Progress")
# Translators: This is the status for a video that the servers have successfully processed
_COMPLETE = gettext_noop("Ready")
# Translators: This is the status for a video that is uploaded completely
_UPLOAD_COMPLETED = gettext_noop("Uploaded")
# Translators: This is the status for a video that the servers have failed to process
_FAILED = gettext_noop("Failed")
# Translators: This is the status for a video that is cancelled during upload by user
_CANCELLED = gettext_noop("Cancelled")
# Translators: This is the status for a video which has failed
# due to being flagged as a duplicate by an external or internal CMS
_DUPLICATE = gettext_noop("Failed Duplicate")
# Translators: This is the status for a video which has duplicate token for youtube
_YOUTUBE_DUPLICATE = gettext_noop("YouTube Duplicate")
# Translators: This is the status for a video for which an invalid
# processing token was provided in the course settings
_INVALID_TOKEN = gettext_noop("Invalid Token")
# Translators: This is the status for a video that was included in a course import
_IMPORTED = gettext_noop("Imported")
# Translators: This is the status for a video that is in an unknown state
_UNKNOWN = gettext_noop("Unknown")
# Translators: This is the status for a video that is having its transcription in progress on servers
_TRANSCRIPTION_IN_PROGRESS = gettext_noop("Transcription in Progress")
# Translators: This is the status for a video whose transcription is complete
_TRANSCRIPT_READY = gettext_noop("Transcript Ready")
# Translators: This is the status for a video whose transcription job was failed for some languages
_PARTIAL_FAILURE = gettext_noop("Partial Failure")
# Translators: This is the status for a video whose transcription job has failed altogether
_TRANSCRIPT_FAILED = gettext_noop("Transcript Failed")
_STATUS_MAP = {
"upload": _UPLOADING,
"ingest": _IN_PROGRESS,
"transcode_queue": _IN_PROGRESS,
"transcode_active": _IN_PROGRESS,
"file_delivered": _COMPLETE,
"file_complete": _COMPLETE,
"upload_completed": _UPLOAD_COMPLETED,
"file_corrupt": _FAILED,
"pipeline_error": _FAILED,
"upload_failed": _FAILED,
"s3_upload_failed": _FAILED,
"upload_cancelled": _CANCELLED,
"duplicate": _DUPLICATE,
"youtube_duplicate": _YOUTUBE_DUPLICATE,
"invalid_token": _INVALID_TOKEN,
"imported": _IMPORTED,
"transcription_in_progress": _TRANSCRIPTION_IN_PROGRESS,
"transcript_ready": _TRANSCRIPT_READY,
"partial_failure": _PARTIAL_FAILURE,
# TODO: Add a related unit tests when the VAL update is part of platform
"transcript_failed": _TRANSCRIPT_FAILED,
}
@staticmethod
def get(val_status):
"""Map a VAL status string to a localized display string"""
# pylint: disable=translation-of-non-string
return _(StatusDisplayStrings._STATUS_MAP.get(val_status, StatusDisplayStrings._UNKNOWN))
@expect_json
@login_required
@require_http_methods(("GET", "POST", "DELETE"))
@@ -199,31 +90,17 @@ def videos_handler(request, course_key_string, edx_video_id=None):
POST
json: create a new video upload; the actual files should not be provided
to this endpoint but rather PUT to the respective upload_url values
contained in the response
contained in the response. Example payload:
{
"files": [{
"file_name": "video.mp4",
"content_type": "video/mp4"
}]
}
DELETE
soft deletes a video for particular course
"""
course = _get_and_validate_course(course_key_string, request.user)
if not course:
return HttpResponseNotFound()
if request.method == "GET":
if "application/json" in request.META.get("HTTP_ACCEPT", ""):
return videos_index_json(course)
pagination_conf = _generate_pagination_configuration(course_key_string, request)
return videos_index_html(course, pagination_conf)
elif request.method == "DELETE":
remove_video_for_course(course_key_string, edx_video_id)
return JsonResponse()
else:
if is_status_update_request(request.json):
return send_video_status_update(request.json)
elif _is_pagination_context_update_request(request):
return _update_pagination_context(request)
data, status = videos_post(course, request)
return JsonResponse(data, status=status)
return handle_videos(request, course_key_string, edx_video_id)
@api_view(['POST'])
@@ -234,12 +111,7 @@ def generate_video_upload_link_handler(request, course_key_string):
API for creating a video upload. Returns an edx_video_id and a presigned URL that can be used
to upload the video to AWS S3.
"""
course = _get_and_validate_course(course_key_string, request.user)
if not course:
return Response(data='Course Not Found', status=rest_status.HTTP_400_BAD_REQUEST)
data, status = videos_post(course, request)
return Response(data, status=status)
return handle_generate_video_upload_link(request, course_key_string)
@expect_json
@@ -247,131 +119,31 @@ def generate_video_upload_link_handler(request, course_key_string):
@require_POST
def video_images_handler(request, course_key_string, edx_video_id=None):
"""Function to handle image files"""
# respond with a 404 if image upload is not enabled.
if not VIDEO_IMAGE_UPLOAD_ENABLED.is_enabled():
return HttpResponseNotFound()
if 'file' not in request.FILES:
return JsonResponse({'error': _('An image file is required.')}, status=400)
image_file = request.FILES['file']
error = validate_video_image(image_file)
if error:
return JsonResponse({'error': error}, status=400)
with closing(image_file):
image_url = update_video_image(edx_video_id, course_key_string, image_file, image_file.name)
LOGGER.info(
'VIDEOS: Video image uploaded for edx_video_id [%s] in course [%s]', edx_video_id, course_key_string
)
return JsonResponse({'image_url': image_url})
return handle_video_images(request, course_key_string, edx_video_id)
@login_required
@require_GET
def video_images_upload_enabled(request):
"""Function to check if images can be uploaded"""
# respond with a false if image upload is not enabled.
if not VIDEO_IMAGE_UPLOAD_ENABLED.is_enabled():
return JsonResponse({'allowThumbnailUpload': False})
return JsonResponse({'allowThumbnailUpload': True})
return check_video_images_upload_enabled(request)
@login_required
@require_GET
def get_video_features(request):
""" Return a dict with info about which video features are enabled """
features = {
'allowThumbnailUpload': VIDEO_IMAGE_UPLOAD_ENABLED.is_enabled(),
'videoSharingEnabled': PUBLIC_VIDEO_SHARE.is_enabled(),
}
return JsonResponse(features)
return enabled_video_features(request)
def validate_transcript_preferences(provider, cielo24_fidelity, cielo24_turnaround,
three_play_turnaround, video_source_language, preferred_languages):
"""
Validate 3rd Party Transcription Preferences.
Arguments:
provider: Transcription provider
cielo24_fidelity: Cielo24 transcription fidelity.
cielo24_turnaround: Cielo24 transcription turnaround.
three_play_turnaround: 3PlayMedia transcription turnaround.
video_source_language: Source/Speech language of the videos that are going to be submitted to the Providers.
preferred_languages: list of language codes.
Returns:
validated preferences or a validation error.
Exposes helper method without breaking existing bindings/dependencies
"""
error, preferences = None, {}
# validate transcription providers
transcription_plans = get_3rd_party_transcription_plans()
if provider in list(transcription_plans.keys()): # lint-amnesty, pylint: disable=consider-iterating-dictionary
# Further validations for providers
if provider == TranscriptProvider.CIELO24:
# Validate transcription fidelity
if cielo24_fidelity in transcription_plans[provider]['fidelity']:
# Validate transcription turnaround
if cielo24_turnaround not in transcription_plans[provider]['turnaround']:
error = f'Invalid cielo24 turnaround {cielo24_turnaround}.'
return error, preferences
# Validate transcription languages
supported_languages = transcription_plans[provider]['fidelity'][cielo24_fidelity]['languages']
if video_source_language not in supported_languages:
error = f'Unsupported source language {video_source_language}.'
return error, preferences
if not preferred_languages or not set(preferred_languages) <= set(supported_languages.keys()):
error = f'Invalid languages {preferred_languages}.'
return error, preferences
# Validated Cielo24 preferences
preferences = {
'video_source_language': video_source_language,
'cielo24_fidelity': cielo24_fidelity,
'cielo24_turnaround': cielo24_turnaround,
'preferred_languages': preferred_languages,
}
else:
error = f'Invalid cielo24 fidelity {cielo24_fidelity}.'
elif provider == TranscriptProvider.THREE_PLAY_MEDIA:
# Validate transcription turnaround
if three_play_turnaround not in transcription_plans[provider]['turnaround']:
error = f'Invalid 3play turnaround {three_play_turnaround}.'
return error, preferences
# Validate transcription languages
valid_translations_map = transcription_plans[provider]['translations']
if video_source_language not in list(valid_translations_map.keys()):
error = f'Unsupported source language {video_source_language}.'
return error, preferences
valid_target_languages = valid_translations_map[video_source_language]
if not preferred_languages or not set(preferred_languages) <= set(valid_target_languages):
error = f'Invalid languages {preferred_languages}.'
return error, preferences
# Validated 3PlayMedia preferences
preferences = {
'three_play_turnaround': three_play_turnaround,
'video_source_language': video_source_language,
'preferred_languages': preferred_languages,
}
else:
error = f'Invalid provider {provider}.'
return error, preferences
return validate_transcript_preferences_source_function(provider, cielo24_fidelity, cielo24_turnaround,
three_play_turnaround, video_source_language,
preferred_languages)
@expect_json
@@ -387,32 +159,7 @@ def transcript_preferences_handler(request, course_key_string):
Returns: valid json response or 400 with error message
"""
course_key = CourseKey.from_string(course_key_string)
is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course_key)
if not is_video_transcript_enabled:
return HttpResponseNotFound()
if request.method == 'POST':
data = request.json
provider = data.get('provider')
error, preferences = validate_transcript_preferences(
provider=provider,
cielo24_fidelity=data.get('cielo24_fidelity', ''),
cielo24_turnaround=data.get('cielo24_turnaround', ''),
three_play_turnaround=data.get('three_play_turnaround', ''),
video_source_language=data.get('video_source_language'),
preferred_languages=list(map(str, data.get('preferred_languages', [])))
)
if error:
response = JsonResponse({'error': error}, status=400)
else:
preferences.update({'provider': provider})
transcript_preferences = create_or_update_transcript_preferences(course_key_string, **preferences)
response = JsonResponse({'transcript_preferences': transcript_preferences}, status=200)
return response
elif request.method == 'DELETE':
remove_transcript_preferences(course_key_string)
return JsonResponse()
return handle_transcript_preferences(request, course_key_string)
@login_required
@@ -425,492 +172,67 @@ def video_encodings_download(request, course_key_string):
Video ID,Name,Status,Profile1 URL,Profile2 URL
aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa,video.mp4,Complete,http://example.com/prof1.mp4,http://example.com/prof2.mp4
"""
course = _get_and_validate_course(course_key_string, request.user)
if not course:
return HttpResponseNotFound()
def get_profile_header(profile):
"""Returns the column header string for the given profile's URLs"""
# Translators: This is the header for a CSV file column
# containing URLs for video encodings for the named profile
# (e.g. desktop, mobile high quality, mobile low quality)
return _("{profile_name} URL").format(profile_name=profile)
profile_whitelist = VideoUploadConfig.get_profile_whitelist()
videos, __ = _get_videos(course)
videos = list(videos)
name_col = _("Name")
duration_col = _("Duration")
added_col = _("Date Added")
video_id_col = _("Video ID")
status_col = _("Status")
profile_cols = [get_profile_header(profile) for profile in profile_whitelist]
def make_csv_dict(video):
"""
Makes a dictionary suitable for writing CSV output. This involves
extracting the required items from the original video dict and
converting all keys and values to UTF-8 encoded string objects,
because the CSV module doesn't play well with unicode objects.
"""
# Translators: This is listed as the duration for a video that has not
# yet reached the point in its processing by the servers where its
# duration is determined.
duration_val = str(video["duration"]) if video["duration"] > 0 else _("Pending")
ret = dict(
[
(name_col, video["client_video_id"]),
(duration_col, duration_val),
(added_col, video["created"].isoformat()),
(video_id_col, video["edx_video_id"]),
(status_col, video["status"]),
] +
[
(get_profile_header(encoded_video["profile"]), encoded_video["url"])
for encoded_video in video["encoded_videos"]
if encoded_video["profile"] in profile_whitelist
]
)
return dict(ret.items())
# Write csv to bytes-like object. We need a separate writer and buffer as the csv
# writer writes str and the FileResponse expects a bytes files.
buffer = io.BytesIO()
buffer_writer = codecs.getwriter("utf-8")(buffer)
writer = csv.DictWriter(
buffer_writer,
[name_col, duration_col, added_col, video_id_col, status_col] + profile_cols,
dialect=csv.excel
)
writer.writeheader()
for video in videos:
writer.writerow(make_csv_dict(video))
buffer.seek(0)
# Translators: This is the suggested filename when downloading the URL
# listing for videos uploaded through Studio
filename = _("{course}_video_urls").format(course=course.id.course) + ".csv"
return FileResponse(buffer, as_attachment=True, filename=filename, content_type="text/csv")
def _get_and_validate_course(course_key_string, user):
"""
Given a course key, return the course if it exists, the given user has
access to it, and it is properly configured for video uploads
"""
course_key = CourseKey.from_string(course_key_string)
# For now, assume all studio users that have access to the course can upload videos.
# In the future, we plan to add a new org-level role for video uploaders.
course = get_course_and_check_access(course_key, user)
if (
settings.FEATURES["ENABLE_VIDEO_UPLOAD_PIPELINE"] and
getattr(settings, "VIDEO_UPLOAD_PIPELINE", None) and
course and
course.video_pipeline_configured
):
return course
else:
return None
return get_video_encodings_download(request, course_key_string)
def convert_video_status(video, is_video_encodes_ready=False):
"""
Convert status of a video. Status can be converted to one of the following:
* FAILED if video is in `upload` state for more than 24 hours
* `YouTube Duplicate` if status is `invalid_token`
* user-friendly video status
Exposes helper method without breaking existing bindings/dependencies
"""
now = datetime.now(video.get('created', datetime.now().replace(tzinfo=UTC)).tzinfo)
if video['status'] == 'upload' and (now - video['created']) > timedelta(hours=MAX_UPLOAD_HOURS):
new_status = 'upload_failed'
status = StatusDisplayStrings.get(new_status)
message = 'Video with id [{}] is still in upload after [{}] hours, setting status to [{}]'.format(
video['edx_video_id'], MAX_UPLOAD_HOURS, new_status
)
send_video_status_update([
{
'edxVideoId': video['edx_video_id'],
'status': new_status,
'message': message
}
])
elif video['status'] == 'invalid_token':
status = StatusDisplayStrings.get('youtube_duplicate')
elif is_video_encodes_ready:
status = StatusDisplayStrings.get('file_complete')
else:
status = StatusDisplayStrings.get(video['status'])
return status
def _get_videos(course, pagination_conf=None):
"""
Retrieves the list of videos from VAL corresponding to this course.
"""
videos, pagination_context = get_videos_for_course(
str(course.id),
VideoSortField.created,
SortDirection.desc,
pagination_conf
)
videos = list(videos)
# This is required to see if edx video pipeline is enabled while converting the video status.
course_video_upload_token = course.video_upload_pipeline.get('course_video_upload_token')
transcription_statuses = ['transcription_in_progress', 'transcript_ready', 'partial_failure', 'transcript_failed']
# convert VAL's status to studio's Video Upload feature status.
for video in videos:
# If we are using "new video workflow" and status is in `transcription_statuses` then video encodes are ready.
# This is because Transcription starts once all the encodes are complete except for YT, but according to
# "new video workflow" YT is disabled as well as deprecated. So, Its precise to say that the Transcription
# starts once all the encodings are complete *for the new video workflow*.
is_video_encodes_ready = not course_video_upload_token and (video['status'] in transcription_statuses)
# Update with transcript languages
video['transcripts'] = get_available_transcript_languages(video_id=video['edx_video_id'])
video['transcription_status'] = (
StatusDisplayStrings.get(video['status']) if is_video_encodes_ready else ''
)
video['transcript_urls'] = {}
for language_code in video['transcripts']:
video['transcript_urls'][language_code] = get_video_transcript_url(
video_id=video['edx_video_id'],
language_code=language_code,
)
# Convert the video status.
video['status'] = convert_video_status(video, is_video_encodes_ready)
return videos, pagination_context
def _get_default_video_image_url():
"""
Returns default video image url
"""
return staticfiles_storage.url(settings.VIDEO_IMAGE_DEFAULT_FILENAME)
def _get_index_videos(course, pagination_conf=None):
"""
Returns the information about each video upload required for the video list
"""
course_id = str(course.id)
attrs = [
'edx_video_id', 'client_video_id', 'created', 'duration',
'status', 'courses', 'transcripts', 'transcription_status',
'transcript_urls', 'error_description'
]
def _get_values(video):
"""
Get data for predefined video attributes.
"""
values = {}
for attr in attrs:
if attr == 'courses':
course = [c for c in video['courses'] if course_id in c]
(__, values['course_video_image_url']), = list(course[0].items())
else:
values[attr] = video[attr]
return values
videos, pagination_context = _get_videos(course, pagination_conf)
return [_get_values(video) for video in videos], pagination_context
return convert_video_status_source_function(video, is_video_encodes_ready)
def get_all_transcript_languages():
"""
Returns all possible languages for transcript.
Exposes helper method without breaking existing bindings/dependencies
"""
third_party_transcription_languages = {}
transcription_plans = get_3rd_party_transcription_plans()
cielo_fidelity = transcription_plans[TranscriptProvider.CIELO24]['fidelity']
# Get third party transcription languages.
third_party_transcription_languages.update(transcription_plans[TranscriptProvider.THREE_PLAY_MEDIA]['languages'])
third_party_transcription_languages.update(cielo_fidelity['MECHANICAL']['languages'])
third_party_transcription_languages.update(cielo_fidelity['PREMIUM']['languages'])
third_party_transcription_languages.update(cielo_fidelity['PROFESSIONAL']['languages'])
all_languages_dict = dict(settings.ALL_LANGUAGES, **third_party_transcription_languages)
# Return combined system settings and 3rd party transcript languages.
all_languages = []
for key, value in sorted(all_languages_dict.items(), key=lambda k_v: k_v[1]):
all_languages.append({
'language_code': key,
'language_text': value
})
return all_languages
return get_all_transcript_languages_source_function()
def videos_index_html(course, pagination_conf=None):
"""
Returns an HTML page to display previous video uploads and allow new ones
Exposes helper method without breaking existing bindings/dependencies
"""
is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course.id)
previous_uploads, pagination_context = _get_index_videos(course, pagination_conf)
context = {
'context_course': course,
'image_upload_url': reverse_course_url('video_images_handler', str(course.id)),
'video_handler_url': reverse_course_url('videos_handler', str(course.id)),
'encodings_download_url': reverse_course_url('video_encodings_download', str(course.id)),
'default_video_image_url': _get_default_video_image_url(),
'previous_uploads': previous_uploads,
'concurrent_upload_limit': settings.VIDEO_UPLOAD_PIPELINE.get('CONCURRENT_UPLOAD_LIMIT', 0),
'video_supported_file_formats': list(VIDEO_SUPPORTED_FILE_FORMATS.keys()),
'video_upload_max_file_size': VIDEO_UPLOAD_MAX_FILE_SIZE_GB,
'video_image_settings': {
'video_image_upload_enabled': VIDEO_IMAGE_UPLOAD_ENABLED.is_enabled(),
'max_size': settings.VIDEO_IMAGE_SETTINGS['VIDEO_IMAGE_MAX_BYTES'],
'min_size': settings.VIDEO_IMAGE_SETTINGS['VIDEO_IMAGE_MIN_BYTES'],
'max_width': settings.VIDEO_IMAGE_MAX_WIDTH,
'max_height': settings.VIDEO_IMAGE_MAX_HEIGHT,
'supported_file_formats': settings.VIDEO_IMAGE_SUPPORTED_FILE_FORMATS
},
'is_video_transcript_enabled': is_video_transcript_enabled,
'active_transcript_preferences': None,
'transcript_credentials': None,
'transcript_available_languages': get_all_transcript_languages(),
'video_transcript_settings': {
'transcript_download_handler_url': reverse('transcript_download_handler'),
'transcript_upload_handler_url': reverse('transcript_upload_handler'),
'transcript_delete_handler_url': reverse_course_url('transcript_delete_handler', str(course.id)),
'trancript_download_file_format': Transcript.SRT
},
'pagination_context': pagination_context
}
if is_video_transcript_enabled:
context['video_transcript_settings'].update({
'transcript_preferences_handler_url': reverse_course_url(
'transcript_preferences_handler',
str(course.id)
),
'transcript_credentials_handler_url': reverse_course_url(
'transcript_credentials_handler',
str(course.id)
),
'transcription_plans': get_3rd_party_transcription_plans(),
})
context['active_transcript_preferences'] = get_transcript_preferences(str(course.id))
# Cached state for transcript providers' credentials (org-specific)
context['transcript_credentials'] = get_transcript_credentials_state_for_org(course.id.org)
if use_new_video_uploads_page(course.id):
return redirect(get_video_uploads_url(course.id))
return render_to_response('videos_index.html', context)
return videos_index_html_source_function(course, pagination_conf)
def videos_index_json(course):
"""
Returns JSON in the following format:
{
'videos': [{
'edx_video_id': 'aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa',
'client_video_id': 'video.mp4',
'created': '1970-01-01T00:00:00Z',
'duration': 42.5,
'status': 'upload',
'course_video_image_url': 'https://video/images/1234.jpg'
}]
}
Exposes helper method without breaking existing bindings/dependencies
"""
index_videos, __ = _get_index_videos(course)
return JsonResponse({"videos": index_videos}, status=200)
return videos_index_json_source_function(course)
def videos_post(course, request):
"""
Input (JSON):
{
"files": [{
"file_name": "video.mp4",
"content_type": "video/mp4"
}]
}
Returns (JSON):
{
"files": [{
"file_name": "video.mp4",
"upload_url": "http://example.com/put_video"
}]
}
The returned array corresponds exactly to the input array.
Exposes helper method without breaking existing bindings/dependencies
"""
error = None
data = request.json
if 'files' not in data:
error = "Request object is not JSON or does not contain 'files'"
elif any(
'file_name' not in file or 'content_type' not in file
for file in data['files']
):
error = "Request 'files' entry does not contain 'file_name' and 'content_type'"
elif any(
file['content_type'] not in list(VIDEO_SUPPORTED_FILE_FORMATS.values())
for file in data['files']
):
error = "Request 'files' entry contain unsupported content_type"
if error:
return {'error': error}, 400
bucket = storage_service_bucket()
req_files = data['files']
resp_files = []
for req_file in req_files:
file_name = req_file['file_name']
try:
file_name.encode('ascii')
except UnicodeEncodeError:
error_msg = 'The file name for %s must contain only ASCII characters.' % file_name
return {'error': error_msg}, 400
edx_video_id = str(uuid4())
key = storage_service_key(bucket, file_name=edx_video_id)
metadata_list = [
('client_video_id', file_name),
('course_key', str(course.id)),
]
course_video_upload_token = course.video_upload_pipeline.get('course_video_upload_token')
# Only include `course_video_upload_token` if youtube has not been deprecated
# for this course.
if not DEPRECATE_YOUTUBE.is_enabled(course.id) and course_video_upload_token:
metadata_list.append(('course_video_upload_token', course_video_upload_token))
is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course.id)
if is_video_transcript_enabled:
transcript_preferences = get_transcript_preferences(str(course.id))
if transcript_preferences is not None:
metadata_list.append(('transcript_preferences', json.dumps(transcript_preferences)))
for metadata_name, value in metadata_list:
key.set_metadata(metadata_name, value)
upload_url = key.generate_url(
KEY_EXPIRATION_IN_SECONDS,
'PUT',
headers={'Content-Type': req_file['content_type']}
)
# persist edx_video_id in VAL
create_video({
'edx_video_id': edx_video_id,
'status': 'upload',
'client_video_id': file_name,
'duration': 0,
'encoded_videos': [],
'courses': [str(course.id)]
})
resp_files.append({'file_name': file_name, 'upload_url': upload_url, 'edx_video_id': edx_video_id})
return {'files': resp_files}, 200
return videos_post_source_function(course, request)
def storage_service_bucket():
"""
Returns an S3 bucket for video upload.
Exposes helper method without breaking existing bindings/dependencies
"""
if ENABLE_DEVSTACK_VIDEO_UPLOADS.is_enabled():
params = {
'aws_access_key_id': settings.AWS_ACCESS_KEY_ID,
'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY,
'security_token': settings.AWS_SECURITY_TOKEN
}
else:
params = {
'aws_access_key_id': settings.AWS_ACCESS_KEY_ID,
'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY
}
conn = S3Connection(**params)
# We don't need to validate our bucket, it requires a very permissive IAM permission
# set since behind the scenes it fires a HEAD request that is equivalent to get_all_keys()
# meaning it would need ListObjects on the whole bucket, not just the path used in each
# environment (since we share a single bucket for multiple deployments in some configurations)
return conn.get_bucket(settings.VIDEO_UPLOAD_PIPELINE['VEM_S3_BUCKET'], validate=False)
return storage_service_bucket_source_function()
def storage_service_key(bucket, file_name):
"""
Returns an S3 key to the given file in the given bucket.
Exposes helper method without breaking existing bindings/dependencies
"""
key_name = "{}/{}".format(
settings.VIDEO_UPLOAD_PIPELINE.get("ROOT_PATH", ""),
file_name
)
return s3.key.Key(bucket, key_name)
return storage_service_key_source_function(bucket, file_name)
def send_video_status_update(updates):
"""
Update video status in edx-val.
Exposes helper method without breaking existing bindings/dependencies
"""
for update in updates:
update_video_status(update.get('edxVideoId'), update.get('status'))
LOGGER.info(
'VIDEOS: Video status update with id [%s], status [%s] and message [%s]',
update.get('edxVideoId'),
update.get('status'),
update.get('message')
)
return JsonResponse()
return send_video_status_update_source_function(updates)
def is_status_update_request(request_data):
"""
Returns True if `request_data` contains status update else False.
Exposes helper method without breaking existing bindings/dependencies
"""
return any('status' in update for update in request_data)
def _generate_pagination_configuration(course_key_string, request):
"""
Returns pagination configuration
"""
course_key = CourseKey.from_string(course_key_string)
if not ENABLE_VIDEO_UPLOAD_PAGINATION.is_enabled(course_key):
return None
return {
'page_number': request.GET.get('page', 1),
'videos_per_page': request.session.get("VIDEOS_PER_PAGE", VIDEOS_PER_PAGE)
}
def _is_pagination_context_update_request(request):
"""
Checks if request contains `videos_per_page`
"""
return request.POST.get('id', '') == "videos_per_page"
def _update_pagination_context(request):
"""
Updates session with posted value
"""
error_msg = _('A non zero positive integer is expected')
try:
videos_per_page = int(request.POST.get('value'))
if videos_per_page <= 0:
return JsonResponse({'error': error_msg}, status=500)
except ValueError:
return JsonResponse({'error': error_msg}, status=500)
request.session['VIDEOS_PER_PAGE'] = videos_per_page
return JsonResponse()
return is_status_update_request_source_function(request_data)

2
cms/urls.py
View File

@@ -328,7 +328,7 @@ from openedx.core.djangoapps.plugins.constants import ProjectType # isort:skip
urlpatterns.extend(get_plugin_url_patterns(ProjectType.CMS))
# Contentstore
# Contentstore REST APIs
urlpatterns += [
path('api/contentstore/', include('cms.djangoapps.contentstore.rest_api.urls'))
]