Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
079f43f40d399cfc0da17039b6dede4e06bda2fb
edx-platform/cms/djangoapps/contentstore/views/transcripts_ajax.py
M. Zulqarnain 41e5403f4e BOM-2369 (D): pyupgrade on contentstore/views (#26767) 
* pyupgrade on contentstore/views

* Apply suggestions from code review

Co-authored-by: Usama Sadiq <usama.sadiq@arbisoft.com>

Co-authored-by: Usama Sadiq <usama.sadiq@arbisoft.com>
2021-03-05 14:55:14 +05:00

661 lines
24 KiB
Python
Raw Blame History

"""
Actions manager for transcripts ajax calls.
+++++++++++++++++++++++++++++++++++++++++++
Module do not support rollback (pressing "Cancel" button in Studio)
All user changes are saved immediately.
"""
import copy
import json
import logging
import os
import requests
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.core.exceptions import PermissionDenied
from django.core.files.base import ContentFile
from django.http import Http404, HttpResponse
from django.utils.translation import ugettext as _
from edxval.api import create_external_video, create_or_update_video_transcript
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import UsageKey
from cms.djangoapps.contentstore.views.videos import TranscriptProvider
from common.djangoapps.student.auth import has_course_author_access
from common.djangoapps.util.json_request import JsonResponse
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
from xmodule.exceptions import NotFoundError
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.video_module.transcripts_utils import (
GetTranscriptsFromYouTubeException,
Transcript,
TranscriptsGenerationException,
TranscriptsRequestValidationException,
clean_video_id,
download_youtube_subs,
get_transcript,
get_transcript_for_video,
get_transcript_from_val,
get_transcripts_from_youtube,
youtube_video_transcript_name
)
__all__ = [
'upload_transcripts',
'download_transcripts',
'check_transcripts',
'choose_transcripts',
'replace_transcripts',
'rename_transcripts',
]
log = logging.getLogger(__name__)
def error_response(response, message, status_code=400):
"""
Simplify similar actions: log message and return JsonResponse with message included in response.
By default return 400 (Bad Request) Response.
"""
log.debug(message)
response['status'] = message
return JsonResponse(response, status_code)
def link_video_to_component(video_component, user):
"""
Links a VAL video to the video component.
Arguments:
video_component: video descriptor item.
user: A requesting user.
Returns:
A cleaned Video ID.
"""
edx_video_id = clean_video_id(video_component.edx_video_id)
if not edx_video_id:
edx_video_id = create_external_video(display_name='external video')
video_component.edx_video_id = edx_video_id
video_component.save_with_metadata(user)
return edx_video_id
def save_video_transcript(edx_video_id, input_format, transcript_content, language_code):
"""
Saves a video transcript to the VAL and its content to the configured django storage(DS).
Arguments:
edx_video_id: A Video ID to associate the transcript.
input_format: Input transcript format for content being passed.
transcript_content: Content of the transcript file
language_code: transcript language code
Returns:
A boolean indicating whether the transcript was saved or not.
"""
try:
# Convert the transcript into the 'sjson' and upload it to
# configured transcript storage. For example, S3.
sjson_subs = Transcript.convert(
content=transcript_content,
input_format=input_format,
output_format=Transcript.SJSON
).encode()
create_or_update_video_transcript(
video_id=edx_video_id,
language_code=language_code,
metadata={
'provider': TranscriptProvider.CUSTOM,
'file_format': Transcript.SJSON,
'language_code': language_code
},
file_data=ContentFile(sjson_subs),
)
result = True
except (TranscriptsGenerationException, UnicodeDecodeError):
result = False
return result
def validate_video_module(request, locator):
"""
Validates video module given its locator and request. Also, checks
if requesting user has course authoring access.
Arguments:
request: WSGI request.
locator: video locator.
Returns:
A tuple containing error(or None) and video descriptor(i.e. if validation succeeds).
Raises:
PermissionDenied: if requesting user does not have access to author the video component.
"""
error, item = None, None
try:
item = _get_item(request, {'locator': locator})
if item.category != 'video':
error = _('Transcripts are supported only for "video" modules.')
except (InvalidKeyError, ItemNotFoundError):
error = _('Cannot find item by locator.')
return error, item
def validate_transcript_upload_data(request):
"""
Validates video transcript file.
Arguments:
request: A WSGI request's data part.
Returns:
Tuple containing an error and validated data
If there is a validation error then, validated data will be empty.
"""
error, validated_data = None, {}
data, files = request.POST, request.FILES
video_locator = data.get('locator')
edx_video_id = data.get('edx_video_id')
if not video_locator:
error = _('Video locator is required.')
elif 'transcript-file' not in files:
error = _('A transcript file is required.')
elif os.path.splitext(files['transcript-file'].name)[1][1:] != Transcript.SRT:
error = _('This transcript file type is not supported.')
elif 'edx_video_id' not in data:
error = _('Video ID is required.')
if not error:
error, video = validate_video_module(request, video_locator)
if not error:
validated_data.update({
'video': video,
'edx_video_id': clean_video_id(edx_video_id) or clean_video_id(video.edx_video_id),
'transcript_file': files['transcript-file']
})
return error, validated_data
@login_required
def upload_transcripts(request):
"""
Upload transcripts for current module.
returns: response dict::
status: 'Success' and HTTP 200 or 'Error' and HTTP 400.
subs: Value of uploaded and saved html5 sub field in video item.
"""
error, validated_data = validate_transcript_upload_data(request)
if error:
response = JsonResponse({'status': error}, status=400)
else:
video = validated_data['video']
edx_video_id = validated_data['edx_video_id']
transcript_file = validated_data['transcript_file']
# check if we need to create an external VAL video to associate the transcript
# and save its ID on the video component.
if not edx_video_id:
edx_video_id = create_external_video(display_name='external video')
video.edx_video_id = edx_video_id
video.save_with_metadata(request.user)
response = JsonResponse({'edx_video_id': edx_video_id, 'status': 'Success'}, status=200)
try:
# Convert 'srt' transcript into the 'sjson' and upload it to
# configured transcript storage. For example, S3.
sjson_subs = Transcript.convert(
content=transcript_file.read().decode('utf-8'),
input_format=Transcript.SRT,
output_format=Transcript.SJSON
).encode()
transcript_created = create_or_update_video_transcript(
video_id=edx_video_id,
language_code='en',
metadata={
'provider': TranscriptProvider.CUSTOM,
'file_format': Transcript.SJSON,
'language_code': 'en'
},
file_data=ContentFile(sjson_subs),
)
if transcript_created is None:
response = JsonResponse({'status': 'Invalid Video ID'}, status=400)
except (TranscriptsGenerationException, UnicodeDecodeError):
response = JsonResponse({
'status': _('There is a problem with this transcript file. Try to upload a different file.')
}, status=400)
return response
@login_required
def download_transcripts(request):
"""
Passes to user requested transcripts file.
Raises Http404 if unsuccessful.
"""
error, video = validate_video_module(request, locator=request.GET.get('locator'))
if error:
raise Http404
try:
content, filename, mimetype = get_transcript(video, lang='en')
except NotFoundError:
raise Http404 # lint-amnesty, pylint: disable=raise-missing-from
# Construct an HTTP response
response = HttpResponse(content, content_type=mimetype)
response['Content-Disposition'] = f'attachment; filename="{filename}"'
return response
@login_required
def check_transcripts(request): # lint-amnesty, pylint: disable=too-many-statements
"""
Check state of transcripts availability.
request.GET['data'] has key `videos`, which can contain any of the following::
[
{u'type': u'youtube', u'video': u'OEoXaMPEzfM', u'mode': u'youtube'},
{u'type': u'html5', u'video': u'video1', u'mode': u'mp4'}
{u'type': u'html5', u'video': u'video2', u'mode': u'webm'}
]
`type` is youtube or html5
`video` is html5 or youtube video_id
`mode` is youtube, ,p4 or webm
Returns transcripts_presence dict::
html5_local: list of html5 ids, if subtitles exist locally for them;
is_youtube_mode: bool, if we have youtube_id, and as youtube mode is of higher priority, reflect this with flag;
youtube_local: bool, if youtube transcripts exist locally;
youtube_server: bool, if youtube transcripts exist on server;
youtube_diff: bool, if youtube transcripts exist on youtube server, and are different from local youtube ones;
current_item_subs: string, value of item.sub field;
status: string, 'Error' or 'Success';
subs: string, new value of item.sub field, that should be set in module;
command: string, action to front-end what to do and what to show to user.
"""
transcripts_presence = {
'html5_local': [],
'html5_equal': False,
'is_youtube_mode': False,
'youtube_local': False,
'youtube_server': False,
'youtube_diff': True,
'current_item_subs': None,
'status': 'Error',
}
try:
__, videos, item = _validate_transcripts_data(request)
except TranscriptsRequestValidationException as e:
return error_response(transcripts_presence, str(e))
transcripts_presence['status'] = 'Success'
try:
edx_video_id = clean_video_id(videos.get('edx_video_id'))
get_transcript_from_val(edx_video_id=edx_video_id, lang='en')
command = 'found'
except NotFoundError:
filename = f'subs_{item.sub}.srt.sjson'
content_location = StaticContent.compute_location(item.location.course_key, filename)
try:
local_transcripts = contentstore().find(content_location).data.decode('utf-8')
transcripts_presence['current_item_subs'] = item.sub
except NotFoundError:
pass
# Check for youtube transcripts presence
youtube_id = videos.get('youtube', None)
if youtube_id:
transcripts_presence['is_youtube_mode'] = True
# youtube local
filename = f'subs_{youtube_id}.srt.sjson'
content_location = StaticContent.compute_location(item.location.course_key, filename)
try:
local_transcripts = contentstore().find(content_location).data.decode('utf-8')
transcripts_presence['youtube_local'] = True
except NotFoundError:
log.debug("Can't find transcripts in storage for youtube id: %s", youtube_id)
# youtube server
youtube_text_api = copy.deepcopy(settings.YOUTUBE['TEXT_API'])
youtube_text_api['params']['v'] = youtube_id
youtube_transcript_name = youtube_video_transcript_name(youtube_text_api)
if youtube_transcript_name:
youtube_text_api['params']['name'] = youtube_transcript_name
youtube_response = requests.get('http://' + youtube_text_api['url'], params=youtube_text_api['params'])
if youtube_response.status_code == 200 and youtube_response.text:
transcripts_presence['youtube_server'] = True
#check youtube local and server transcripts for equality
if transcripts_presence['youtube_server'] and transcripts_presence['youtube_local']:
try:
youtube_server_subs = get_transcripts_from_youtube(
youtube_id,
settings,
item.runtime.service(item, "i18n")
)
if json.loads(local_transcripts) == youtube_server_subs: # check transcripts for equality
transcripts_presence['youtube_diff'] = False
except GetTranscriptsFromYouTubeException:
pass
# Check for html5 local transcripts presence
html5_subs = []
for html5_id in videos['html5']:
filename = f'subs_{html5_id}.srt.sjson'
content_location = StaticContent.compute_location(item.location.course_key, filename)
try:
html5_subs.append(contentstore().find(content_location).data)
transcripts_presence['html5_local'].append(html5_id)
except NotFoundError:
log.debug("Can't find transcripts in storage for non-youtube video_id: %s", html5_id)
if len(html5_subs) == 2: # check html5 transcripts for equality
transcripts_presence['html5_equal'] = (
json.loads(html5_subs[0].decode('utf-8')) == json.loads(html5_subs[1].decode('utf-8'))
)
command, __ = _transcripts_logic(transcripts_presence, videos)
transcripts_presence.update({'command': command})
return JsonResponse(transcripts_presence)
def _transcripts_logic(transcripts_presence, videos):
"""
By `transcripts_presence` content, figure what show to user:
returns: `command` and `subs`.
`command`: string, action to front-end what to do and what show to user.
`subs`: string, new value of item.sub field, that should be set in module.
`command` is one of::
replace: replace local youtube subtitles with server one's
found: subtitles are found
import: import subtitles from youtube server
choose: choose one from two html5 subtitles
not found: subtitles are not found
"""
command = None
# new value of item.sub field, that should be set in module.
subs = ''
# youtube transcripts are of high priority than html5 by design
if (
transcripts_presence['youtube_diff'] and
transcripts_presence['youtube_local'] and
transcripts_presence['youtube_server']): # youtube server and local exist
command = 'replace'
subs = videos['youtube']
elif transcripts_presence['youtube_local']: # only youtube local exist
command = 'found'
subs = videos['youtube']
elif transcripts_presence['youtube_server']: # only youtube server exist
command = 'import'
else: # html5 part
if transcripts_presence['html5_local']: # can be 1 or 2 html5 videos
if len(transcripts_presence['html5_local']) == 1 or transcripts_presence['html5_equal']:
command = 'found'
subs = transcripts_presence['html5_local'][0]
else:
command = 'choose'
subs = transcripts_presence['html5_local'][0]
else: # html5 source have no subtitles
# check if item sub has subtitles
if transcripts_presence['current_item_subs'] and not transcripts_presence['is_youtube_mode']:
log.debug("Command is use existing %s subs", transcripts_presence['current_item_subs'])
command = 'use_existing'
else:
command = 'not_found'
log.debug(
"Resulted command: %s, current transcripts: %s, youtube mode: %s",
command,
transcripts_presence['current_item_subs'],
transcripts_presence['is_youtube_mode']
)
return command, subs
def _validate_transcripts_data(request):
"""
Validates, that request contains all proper data for transcripts processing.
Returns tuple of 3 elements::
data: dict, loaded json from request,
videos: parsed `data` to useful format,
item: video item from storage
Raises `TranscriptsRequestValidationException` if validation is unsuccessful
or `PermissionDenied` if user has no access.
"""
data = json.loads(request.GET.get('data', '{}'))
if not data:
raise TranscriptsRequestValidationException(_('Incoming video data is empty.'))
try:
item = _get_item(request, data)
except (InvalidKeyError, ItemNotFoundError):
raise TranscriptsRequestValidationException(_("Can't find item by locator.")) # lint-amnesty, pylint: disable=raise-missing-from
if item.category != 'video':
raise TranscriptsRequestValidationException(_('Transcripts are supported only for "video" modules.'))
# parse data form request.GET.['data']['video'] to useful format
videos = {'youtube': '', 'html5': {}}
for video_data in data.get('videos'):
if video_data['type'] == 'youtube':
videos['youtube'] = video_data['video']
elif video_data['type'] == 'edx_video_id':
if clean_video_id(video_data['video']):
videos['edx_video_id'] = video_data['video']
else: # do not add same html5 videos
if videos['html5'].get('video') != video_data['video']:
videos['html5'][video_data['video']] = video_data['mode']
return data, videos, item
def validate_transcripts_request(request, include_yt=False, include_html5=False):
"""
Validates transcript handler's request.
NOTE: This is one central validation flow for `choose_transcripts`,
`check_transcripts` and `replace_transcripts` handlers.
Returns:
A tuple containing:
1. An error message in case of validation failure.
2. validated video data
"""
error = None
validated_data = {'video': None, 'youtube': '', 'html5': {}}
# Loads the request data
data = json.loads(request.GET.get('data', '{}'))
if not data:
error = _('Incoming video data is empty.')
else:
error, video = validate_video_module(request, locator=data.get('locator'))
if not error:
validated_data.update({'video': video})
videos = data.get('videos', [])
if include_yt:
validated_data.update({
video['type']: video['video']
for video in videos
if video['type'] == 'youtube'
})
if include_html5:
validated_data['chosen_html5_id'] = data.get('html5_id')
validated_data['html5'] = {
video['video']: video['mode']
for video in videos
if video['type'] != 'youtube'
}
return error, validated_data
@login_required
def choose_transcripts(request):
"""
Create/Update edx transcript in DS with chosen html5 subtitles from contentstore.
Returns:
status `Success` and resulted `edx_video_id` value
Or error in case of validation failures.
"""
error, validated_data = validate_transcripts_request(request, include_html5=True)
if error:
response = error_response({}, error)
else:
# 1. Retrieve transcript file for `chosen_html5_id` from contentstore.
try:
video = validated_data['video']
chosen_html5_id = validated_data['chosen_html5_id']
input_format, __, transcript_content = get_transcript_for_video(
video.location,
subs_id=chosen_html5_id,
file_name=chosen_html5_id,
language='en'
)
except NotFoundError:
return error_response({}, _('No such transcript.'))
# 2. Link a video to video component if its not already linked to one.
edx_video_id = link_video_to_component(video, request.user)
# 3. Upload the retrieved transcript to DS for the linked video ID.
success = save_video_transcript(edx_video_id, input_format, transcript_content, language_code='en')
if success:
response = JsonResponse({'edx_video_id': edx_video_id, 'status': 'Success'}, status=200)
else:
response = error_response({}, _('There is a problem with the chosen transcript file.'))
return response
@login_required
def rename_transcripts(request):
"""
Copies existing transcript on video component's `sub`(from contentstore) into the
DS for a video.
Returns:
status `Success` and resulted `edx_video_id` value
Or error in case of validation failures.
"""
error, validated_data = validate_transcripts_request(request)
if error:
response = error_response({}, error)
else:
# 1. Retrieve transcript file for `video.sub` from contentstore.
try:
video = validated_data['video']
input_format, __, transcript_content = get_transcript_for_video(
video.location,
subs_id=video.sub,
file_name=video.sub,
language='en'
)
except NotFoundError:
return error_response({}, _('No such transcript.'))
# 2. Link a video to video component if its not already linked to one.
edx_video_id = link_video_to_component(video, request.user)
# 3. Upload the retrieved transcript to DS for the linked video ID.
success = save_video_transcript(edx_video_id, input_format, transcript_content, language_code='en')
if success:
response = JsonResponse({'edx_video_id': edx_video_id, 'status': 'Success'}, status=200)
else:
response = error_response(
{}, _('There is a problem with the existing transcript file. Please upload a different file.')
)
return response
@login_required
def replace_transcripts(request):
"""
Downloads subtitles from youtube and replaces edx transcripts in DS with youtube ones.
Returns:
status `Success` and resulted `edx_video_id` value
Or error on validation failures.
"""
error, validated_data = validate_transcripts_request(request, include_yt=True)
youtube_id = validated_data['youtube']
if error:
response = error_response({}, error)
elif not youtube_id:
response = error_response({}, _('YouTube ID is required.'))
else:
# 1. Download transcript from YouTube.
try:
video = validated_data['video']
transcript_content = download_youtube_subs(youtube_id, video, settings)
except GetTranscriptsFromYouTubeException as e:
return error_response({}, str(e))
# 2. Link a video to video component if its not already linked to one.
edx_video_id = link_video_to_component(video, request.user)
# 3. Upload YT transcript to DS for the linked video ID.
success = save_video_transcript(edx_video_id, Transcript.SJSON, transcript_content, language_code='en')
if success:
response = JsonResponse({'edx_video_id': edx_video_id, 'status': 'Success'}, status=200)
else:
response = error_response({}, _('There is a problem with the YouTube transcript file.'))
return response
def _get_item(request, data):
"""
Obtains from 'data' the locator for an item.
Next, gets that item from the modulestore (allowing any errors to raise up).
Finally, verifies that the user has access to the item.
Returns the item.
"""
usage_key = UsageKey.from_string(data.get('locator'))
# This is placed before has_course_author_access() to validate the location,
# because has_course_author_access() raises r if location is invalid.
item = modulestore().get_item(usage_key)
# use the item's course_key, because the usage_key might not have the run
if not has_course_author_access(request.user, item.location.course_key):
raise PermissionDenied()
return item
Reference in New Issue View Git Blame Copy Permalink