From c90aa33cb8dc17dc835c46c43039589659137ac1 Mon Sep 17 00:00:00 2001
From: Qubad786 <muhammadrehan69@gmail.com>
Date: Wed, 14 Feb 2018 19:23:04 +0500
Subject: [PATCH] Integrate with the updated VAL api utils.

---
 .../tests/test_transcripts_utils.py           |  4 +-
 .../views/tests/test_transcript_settings.py   |  2 +-
 .../views/tests/test_transcripts.py           |  4 +-
 .../contentstore/views/transcript_settings.py |  6 +-
 .../contentstore/views/transcripts_ajax.py    | 24 +++----
 cms/djangoapps/contentstore/views/videos.py   |  2 +-
 .../xmodule/video_module/transcripts_utils.py | 66 ++++++++++++-------
 .../xmodule/video_module/video_handlers.py    | 38 ++++-------
 .../courseware/tests/test_video_handlers.py   |  9 ++-
 .../courseware/tests/test_video_mongo.py      |  4 +-
 .../mobile_api/video_outlines/views.py        | 17 ++---
 requirements/edx/base.txt                     |  2 +-
 requirements/edx/github.txt                   |  1 +
 13 files changed, 89 insertions(+), 90 deletions(-)

diff --git a/cms/djangoapps/contentstore/tests/test_transcripts_utils.py b/cms/djangoapps/contentstore/tests/test_transcripts_utils.py
index 84e9a497c9..ae2db05e31 100644
--- a/cms/djangoapps/contentstore/tests/test_transcripts_utils.py
+++ b/cms/djangoapps/contentstore/tests/test_transcripts_utils.py
@@ -647,9 +647,9 @@ class TestTranscript(unittest.TestCase):
         """
         Tests that the srt transcript is successfully converted into sjson format.
         """
-        expected = json.loads(self.sjson_transcript)
+        expected = self.sjson_transcript
         actual = transcripts_utils.Transcript.convert(self.srt_transcript, 'srt', 'sjson')
-        self.assertDictEqual(actual, expected)
+        self.assertDictEqual(json.loads(actual), json.loads(expected))
 
     def test_convert_invalid_srt_to_sjson(self):
         """
diff --git a/cms/djangoapps/contentstore/views/tests/test_transcript_settings.py b/cms/djangoapps/contentstore/views/tests/test_transcript_settings.py
index 40174725b4..443131af6b 100644
--- a/cms/djangoapps/contentstore/views/tests/test_transcript_settings.py
+++ b/cms/djangoapps/contentstore/views/tests/test_transcript_settings.py
@@ -578,4 +578,4 @@ class TranscriptUploadTest(CourseTestCase):
             language_code=language_code
         ))
         self.assertEqual(response.status_code, 200)
-        self.assertFalse(api.get_video_transcript_data([video_id], language_code=language_code))
+        self.assertFalse(api.get_video_transcript_data(video_id=video_id, language_code=language_code))
diff --git a/cms/djangoapps/contentstore/views/tests/test_transcripts.py b/cms/djangoapps/contentstore/views/tests/test_transcripts.py
index 0db2f2c549..9d99c072d9 100644
--- a/cms/djangoapps/contentstore/views/tests/test_transcripts.py
+++ b/cms/djangoapps/contentstore/views/tests/test_transcripts.py
@@ -542,7 +542,7 @@ class TestDownloadTranscripts(BaseTranscripts):
         }
 
         self.item.data = textwrap.dedent("""
-            <video youtube="" sub="">
+            <video youtube="" sub="" edx_video_id="123">
                 <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                 <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                 <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
@@ -838,7 +838,7 @@ class TestCheckTranscripts(BaseTranscripts):
         """
         video_transcript_feature.return_value = feature_enabled
         self.item.data = textwrap.dedent("""
-            <video youtube="" sub="">
+            <video youtube="" sub="" edx_video_id="123">
                 <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                 <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                 <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
diff --git a/cms/djangoapps/contentstore/views/transcript_settings.py b/cms/djangoapps/contentstore/views/transcript_settings.py
index 6ed21e03b2..cb891107fb 100644
--- a/cms/djangoapps/contentstore/views/transcript_settings.py
+++ b/cms/djangoapps/contentstore/views/transcript_settings.py
@@ -156,7 +156,7 @@ def transcript_download_handler(request, course_key_string):
 
     edx_video_id = request.GET['edx_video_id']
     language_code = request.GET['language_code']
-    transcript = get_video_transcript_data(video_ids=[edx_video_id], language_code=language_code)
+    transcript = get_video_transcript_data(video_id=edx_video_id, language_code=language_code)
     if transcript:
         name_and_extension = os.path.splitext(transcript['file_name'])
         basename, file_format = name_and_extension[0], name_and_extension[1][1:]
@@ -193,7 +193,7 @@ def validate_transcript_upload_data(data, files):
         error = _(u'The following parameters are required: {missing}.').format(missing=', '.join(missing))
     elif (
         data['language_code'] != data['new_language_code'] and
-        data['new_language_code'] in get_available_transcript_languages([data['edx_video_id']])
+        data['new_language_code'] in get_available_transcript_languages(video_id=data['edx_video_id'])
     ):
         error = _(u'A transcript with the "{language_code}" language code already exists.'.format(
             language_code=data['new_language_code']
@@ -251,7 +251,7 @@ def transcript_upload_handler(request, course_key_string):
                     'file_format': Transcript.SJSON,
                     'language_code': new_language_code
                 },
-                file_data=ContentFile(json.dumps(sjson_subs)),
+                file_data=ContentFile(sjson_subs),
             )
             response = JsonResponse(status=201)
         except (TranscriptsGenerationException, UnicodeDecodeError):
diff --git a/cms/djangoapps/contentstore/views/transcripts_ajax.py b/cms/djangoapps/contentstore/views/transcripts_ajax.py
index 247f7e798c..1c44a23391 100644
--- a/cms/djangoapps/contentstore/views/transcripts_ajax.py
+++ b/cms/djangoapps/contentstore/views/transcripts_ajax.py
@@ -173,26 +173,23 @@ def download_transcripts(request):
             item.location.course_key,
             'subs_{filename}.srt.sjson'.format(filename=filename),
         )
-        sjson_transcript = contentstore().find(content_location).data
+        input_format = Transcript.SJSON
+        transcript_content = contentstore().find(content_location).data
     except NotFoundError:
         # Try searching in VAL for the transcript as a last resort
         transcript = None
         if is_val_transcript_feature_enabled_for_course(item.location.course_key):
-            transcript = get_video_transcript_content(
-                language_code=u'en',
-                edx_video_id=item.edx_video_id,
-                youtube_id_1_0=item.youtube_id_1_0,
-                html5_sources=item.html5_sources,
-            )
+            transcript = get_video_transcript_content(edx_video_id=item.edx_video_id, language_code=u'en')
 
         if not transcript:
             raise Http404
 
-        filename = os.path.splitext(os.path.basename(transcript['file_name']))[0].encode('utf8')
-        sjson_transcript = transcript['content']
+        name_and_extension = os.path.splitext(transcript['file_name'])
+        filename, input_format = name_and_extension[0], name_and_extension[1][1:]
+        transcript_content = transcript['content']
 
     # convert sjson content into srt format.
-    transcript_content = Transcript.convert(sjson_transcript, input_format='sjson', output_format='srt')
+    transcript_content = Transcript.convert(transcript_content, input_format=input_format, output_format=Transcript.SRT)
     if not transcript_content:
         raise Http404
 
@@ -309,12 +306,7 @@ def check_transcripts(request):
     if command == 'not_found':
         # Try searching in VAL for the transcript as a last resort
         if is_val_transcript_feature_enabled_for_course(item.location.course_key):
-            video_transcript = get_video_transcript_content(
-                language_code=u'en',
-                edx_video_id=item.edx_video_id,
-                youtube_id_1_0=item.youtube_id_1_0,
-                html5_sources=item.html5_sources,
-            )
+            video_transcript = get_video_transcript_content(edx_video_id=item.edx_video_id, language_code=u'en')
             command = 'found' if video_transcript else command
 
     transcripts_presence.update({
diff --git a/cms/djangoapps/contentstore/views/videos.py b/cms/djangoapps/contentstore/views/videos.py
index d29ba6deb9..2844d9ddd0 100644
--- a/cms/djangoapps/contentstore/views/videos.py
+++ b/cms/djangoapps/contentstore/views/videos.py
@@ -541,7 +541,7 @@ def _get_videos(course):
         video["status"] = convert_video_status(video)
 
         if is_video_transcript_enabled:
-            video['transcripts'] = get_available_transcript_languages([video['edx_video_id']])
+            video['transcripts'] = get_available_transcript_languages(video_id=video['edx_video_id'])
 
     return videos
 
diff --git a/common/lib/xmodule/xmodule/video_module/transcripts_utils.py b/common/lib/xmodule/xmodule/video_module/transcripts_utils.py
index e49ae981c9..e831a67e93 100644
--- a/common/lib/xmodule/xmodule/video_module/transcripts_utils.py
+++ b/common/lib/xmodule/xmodule/video_module/transcripts_utils.py
@@ -527,47 +527,73 @@ def get_video_ids_info(edx_video_id, youtube_id_1_0, html5_sources):
     return external, video_ids
 
 
-def get_video_transcript_content(language_code, edx_video_id, youtube_id_1_0, html5_sources):
+def clean_video_id(edx_video_id):
+    """
+    Cleans an edx video ID.
+
+    Arguments:
+        edx_video_id(unicode): edx-val's video identifier
+    """
+    return edx_video_id and edx_video_id.strip()
+
+
+def get_video_transcript_content(edx_video_id, language_code):
     """
     Gets video transcript content, only if the corresponding feature flag is enabled for the given `course_id`.
 
     Arguments:
         language_code(unicode): Language code of the requested transcript
         edx_video_id(unicode): edx-val's video identifier
-        youtube_id_1_0(unicode): A youtube source identifier
-        html5_sources(list): A list containing html5 sources
 
     Returns:
         A dict containing transcript's file name and its sjson content.
     """
     transcript = None
-    if edxval_api:
-        __, video_candidate_ids = get_video_ids_info(edx_video_id, youtube_id_1_0, html5_sources)
-        transcript = edxval_api.get_video_transcript_data(video_candidate_ids, language_code)
+    edx_video_id = clean_video_id(edx_video_id)
+    if edxval_api and edx_video_id:
+        transcript = edxval_api.get_video_transcript_data(edx_video_id, language_code)
 
     return transcript
 
 
-def get_available_transcript_languages(edx_video_id, youtube_id_1_0, html5_sources):
+def get_available_transcript_languages(edx_video_id):
     """
-    Gets available transcript languages from edx-val.
+    Gets available transcript languages for a video.
 
     Arguments:
         edx_video_id(unicode): edx-val's video identifier
-        youtube_id_1_0(unicode): A youtube source identifier
-        html5_sources(list): A list containing html5 sources
 
     Returns:
         A list containing distinct transcript language codes against all the passed video ids.
     """
     available_languages = []
-    if edxval_api:
-        __, video_candidate_ids = get_video_ids_info(edx_video_id, youtube_id_1_0, html5_sources)
-        available_languages = edxval_api.get_available_transcript_languages(video_candidate_ids)
+    edx_video_id = clean_video_id(edx_video_id)
+    if edxval_api and edx_video_id:
+        available_languages = edxval_api.get_available_transcript_languages(video_id=edx_video_id)
 
     return available_languages
 
 
+def convert_video_transcript(file_name, content, output_format):
+    """
+    Convert video transcript into desired format
+
+    Arguments:
+        file_name: name of transcript file along with its extension
+        content: transcript content stream
+        output_format: the format in which transcript will be converted
+
+    Returns:
+        A dict containing the new transcript filename and the content converted into desired format.
+    """
+    name_and_extension = os.path.splitext(file_name)
+    basename, input_format = name_and_extension[0], name_and_extension[1][1:]
+    filename = '{base_name}.{ext}'.format(base_name=basename.encode('utf8'), ext=output_format)
+    converted_transcript = Transcript.convert(content, input_format=input_format, output_format=output_format)
+
+    return dict(filename=filename, content=converted_transcript)
+
+
 class Transcript(object):
     """
     Container for transcript methods.
@@ -615,7 +641,7 @@ class Transcript(object):
                 except Error as ex:   # Base exception from pysrt
                     raise TranscriptsGenerationException(text_type(ex))
 
-                return generate_sjson_from_srt(srt_subs)
+                return json.dumps(generate_sjson_from_srt(srt_subs))
 
         if input_format == 'sjson':
 
@@ -712,11 +738,7 @@ class VideoTranscriptsMixin(object):
         # If we've gotten this far, we're going to verify that the transcripts
         # being referenced are actually either in the contentstore or in edx-val.
         if include_val_transcripts:
-            translations = get_available_transcript_languages(
-                edx_video_id=self.edx_video_id,
-                youtube_id_1_0=self.youtube_id_1_0,
-                html5_sources=self.html5_sources
-            )
+            translations = get_available_transcript_languages(edx_video_id=self.edx_video_id)
 
         if sub:  # check if sjson exists for 'en'.
             try:
@@ -827,11 +849,7 @@ class VideoTranscriptsMixin(object):
         # For phase 2, removing `include_val_transcripts` will make edx-val
         # taking over the control for transcripts.
         if include_val_transcripts:
-            transcript_languages = get_available_transcript_languages(
-                edx_video_id=self.edx_video_id,
-                youtube_id_1_0=self.youtube_id_1_0,
-                html5_sources=self.html5_sources
-            )
+            transcript_languages = get_available_transcript_languages(edx_video_id=self.edx_video_id)
             # HACK Warning! this is temporary and will be removed once edx-val take over the
             # transcript module and contentstore will only function as fallback until all the
             # data is migrated to edx-val.
diff --git a/common/lib/xmodule/xmodule/video_module/video_handlers.py b/common/lib/xmodule/xmodule/video_module/video_handlers.py
index 2212ed6e9d..5b5c57dbf3 100644
--- a/common/lib/xmodule/xmodule/video_module/video_handlers.py
+++ b/common/lib/xmodule/xmodule/video_module/video_handlers.py
@@ -21,6 +21,7 @@ from xmodule.fields import RelativeTime
 from opaque_keys.edx.locator import CourseLocator
 
 from .transcripts_utils import (
+    convert_video_transcript,
     get_or_create_sjson,
     generate_sjson_for_all_speeds,
     get_video_transcript_content,
@@ -297,19 +298,16 @@ class VideoStudentViewHandlers(object):
                 response = self.get_static_transcript(request, transcripts)
                 if response.status_code == 404 and feature_enabled:
                     # Try to get transcript from edx-val as a last resort.
-                    transcript = get_video_transcript_content(
-                        language_code=self.transcript_language,
-                        edx_video_id=self.edx_video_id,
-                        youtube_id_1_0=self.youtube_id_1_0,
-                        html5_sources=self.html5_sources,
-                    )
+                    transcript = get_video_transcript_content(self.edx_video_id, self.transcript_language)
                     if transcript:
+                        transcript_conversion_props = dict(transcript, output_format=Transcript.SJSON)
+                        transcript = convert_video_transcript(**transcript_conversion_props)
                         response = Response(
                             transcript['content'],
                             headerlist=[('Content-Language', self.transcript_language)],
                             charset='utf8',
                         )
-                        response.content_type = Transcript.mime_types['sjson']
+                        response.content_type = Transcript.mime_types[Transcript.SJSON]
 
                 return response
             except (UnicodeDecodeError, TranscriptsGenerationException) as ex:
@@ -335,28 +333,16 @@ class VideoStudentViewHandlers(object):
                     if not lang:
                         lang = self.get_default_transcript_language(transcripts)
 
-                    transcript = get_video_transcript_content(
-                        language_code=lang,
-                        edx_video_id=self.edx_video_id,
-                        youtube_id_1_0=self.youtube_id_1_0,
-                        html5_sources=self.html5_sources,
-                    )
+                    transcript = get_video_transcript_content(edx_video_id=self.edx_video_id, language_code=lang)
                     if transcript:
-                        transcript_content = Transcript.convert(
-                            transcript['content'],
-                            input_format='sjson',
-                            output_format=self.transcript_download_format
-                        )
-                        # Construct the response
-                        base_name, __ = os.path.splitext(os.path.basename(transcript['file_name']))
-                        filename = '{base_name}.{ext}'.format(
-                            base_name=base_name.encode('utf8'),
-                            ext=self.transcript_download_format
-                        )
+                        transcript_conversion_props = dict(transcript, output_format=self.transcript_download_format)
+                        transcript = convert_video_transcript(**transcript_conversion_props)
                         response = Response(
-                            transcript_content,
+                            transcript['content'],
                             headerlist=[
-                                ('Content-Disposition', 'attachment; filename="{filename}"'.format(filename=filename)),
+                                ('Content-Disposition', 'attachment; filename="{filename}"'.format(
+                                    filename=transcript['filename']
+                                )),
                                 ('Content-Language', lang),
                             ],
                             charset='utf8',
diff --git a/lms/djangoapps/courseware/tests/test_video_handlers.py b/lms/djangoapps/courseware/tests/test_video_handlers.py
index 845cfde03d..4f6b7a99db 100644
--- a/lms/djangoapps/courseware/tests/test_video_handlers.py
+++ b/lms/djangoapps/courseware/tests/test_video_handlers.py
@@ -413,7 +413,8 @@ class TestTranscriptDownloadDispatch(TestVideo):
     DATA = """
         <video show_captions="true"
         display_name="A Name"
-        sub='OEoXaMPEzfM'
+        sub="OEoXaMPEzfM"
+        edx_video_id="123"
         >
             <source src="example.mp4"/>
             <source src="example.webm"/>
@@ -528,8 +529,10 @@ class TestTranscriptTranslationGetDispatch(TestVideo):
 
     srt_file = _create_srt_file()
     DATA = """
-        <video show_captions="true"
-        display_name="A Name"
+        <video
+            show_captions="true"
+            display_name="A Name"
+            edx_video_id="123"
         >
             <source src="example.mp4"/>
             <source src="example.webm"/>
diff --git a/lms/djangoapps/courseware/tests/test_video_mongo.py b/lms/djangoapps/courseware/tests/test_video_mongo.py
index 1c7075cd58..ad7314441c 100644
--- a/lms/djangoapps/courseware/tests/test_video_mongo.py
+++ b/lms/djangoapps/courseware/tests/test_video_mongo.py
@@ -1310,7 +1310,9 @@ class TestVideoDescriptorStudentViewJson(TestCase):
 
     def setUp(self):
         super(TestVideoDescriptorStudentViewJson, self).setUp()
-        video_declaration = "<video display_name='Test Video' youtube_id_1_0=\'" + self.TEST_YOUTUBE_ID + "\'>"
+        video_declaration = (
+            "<video display_name='Test Video' edx_video_id='123' youtube_id_1_0=\'" + self.TEST_YOUTUBE_ID + "\'>"
+        )
         sample_xml = ''.join([
             video_declaration,
             "<source src='", self.TEST_SOURCE_URL, "'/> ",
diff --git a/lms/djangoapps/mobile_api/video_outlines/views.py b/lms/djangoapps/mobile_api/video_outlines/views.py
index 57b919388b..46314ed7a1 100644
--- a/lms/djangoapps/mobile_api/video_outlines/views.py
+++ b/lms/djangoapps/mobile_api/video_outlines/views.py
@@ -18,6 +18,7 @@ from mobile_api.models import MobileApiConfig
 from xmodule.exceptions import NotFoundError
 from xmodule.modulestore.django import modulestore
 from xmodule.video_module.transcripts_utils import (
+    convert_video_transcript,
     get_video_transcript_content,
     Transcript,
 )
@@ -129,20 +130,16 @@ class VideoTranscripts(generics.RetrieveAPIView):
             # Fallback mechanism for edx-val transcripts
             transcript = None
             if feature_enabled:
-                transcript = get_video_transcript_content(
-                    language_code=lang,
-                    edx_video_id=video_descriptor.edx_video_id,
-                    youtube_id_1_0=video_descriptor.youtube_id_1_0,
-                    html5_sources=video_descriptor.html5_sources,
-                )
+                transcript = get_video_transcript_content(video_descriptor.edx_video_id, lang)
 
             if not transcript:
                 raise Http404(u'Transcript not found for {}, lang: {}'.format(block_id, lang))
 
-            base_name, __ = os.path.splitext(os.path.basename(transcript['file_name']))
-            filename = '{base_name}.srt'.format(base_name=base_name)
-            content = Transcript.convert(transcript['content'], 'sjson', 'srt')
-            mimetype = Transcript.mime_types['srt']
+            transcript_conversion_props = dict(transcript, output_format=Transcript.SRT)
+            transcript = convert_video_transcript(**transcript_conversion_props)
+            filename = transcript['filename']
+            content = transcript['content']
+            mimetype = Transcript.mime_types[Transcript.SRT]
         except KeyError:
             raise Http404(u"Transcript not found for {}, lang: {}".format(block_id, lang))
 
diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt
index 975eba909e..dde335963c 100644
--- a/requirements/edx/base.txt
+++ b/requirements/edx/base.txt
@@ -66,7 +66,7 @@ edx-rest-api-client==1.7.1
 edx-search==1.1.0
 edx-submissions==2.0.12
 edx-user-state-client==1.0.3
-edxval==0.1.10
+#edxval==0.1.10
 event-tracking==0.2.4
 feedparser==5.1.3
 firebase-token-generator==1.3.2
diff --git a/requirements/edx/github.txt b/requirements/edx/github.txt
index 603338c59c..04343b2c7f 100644
--- a/requirements/edx/github.txt
+++ b/requirements/edx/github.txt
@@ -100,6 +100,7 @@ git+https://github.com/solashirai/crowdsourcehinter.git@518605f0a95190949fe77bd3
 git+https://github.com/edx/xblock-utils.git@v1.0.5#egg=xblock-utils==1.0.5
 -e git+https://github.com/edx-solutions/xblock-google-drive.git@138e6fa0bf3a2013e904a085b9fed77dab7f3f21#egg=xblock-google-drive
 git+https://github.com/edx/xblock-lti-consumer.git@v1.1.7#egg=lti_consumer-xblock==1.1.7
+-e git+https://github.com/edx/edx-val.git@mrehan/refactor-val-api-utils#egg=edxval
 
 # Third Party XBlocks