Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add video thumbnails management command to scrape youtube thumbnails and update them in edxval

Browse Source
This commit is contained in:
irfanuddinahmad
2018-07-31 17:08:55 +05:00
committed by M. Rehan
parent 89925a96dc
commit 527edc5c45
12 changed files with 459 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
cms/djangoapps/contentstore/management/commands/tests/test_video_thumbnails.py Normal file
View File

@@ -0,0 +1,137 @@
# -*- coding: utf-8 -*-
"""
Tests for course video thumbnails management command.
"""
import logging
from mock import patch
from django.core.management import call_command, CommandError
from django.test import TestCase
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from openedx.core.djangoapps.video_config.models import VideoThumbnailSetting
from six import text_type
from testfixtures import LogCapture
LOGGER_NAME = "contentstore.management.commands.video_thumbnails"
def setup_video_thumbnails_config(batch_size=10, commit=False, all_course_videos=False, course_ids=''):
VideoThumbnailSetting.objects.create(
batch_size=batch_size,
commit=commit,
course_ids=course_ids,
all_course_videos=all_course_videos,
videos_per_task=2
)
class TestArgParsing(TestCase):
"""
Tests for parsing arguments for the `video_thumbnails` management command
"""
def test_invalid_course(self):
errstring = "Invalid key specified: <class 'opaque_keys.edx.locator.CourseLocator'>: invalid-course"
setup_video_thumbnails_config(course_ids='invalid-course')
with self.assertRaisesRegexp(CommandError, errstring):
call_command('video_thumbnails')
class TestVideoThumbnails(ModuleStoreTestCase):
"""
Tests adding thumbnails to course videos from YouTube
"""
def setUp(self):
""" Common setup """
super(TestVideoThumbnails, self).setUp()
self.course = CourseFactory.create()
self.course_2 = CourseFactory.create()
@patch('edxval.api.get_course_video_ids_with_youtube_profile')
@patch('contentstore.management.commands.video_thumbnails.enqueue_update_thumbnail_tasks')
def test_video_thumbnails_without_commit(self, mock_enqueue_thumbnails, mock_course_videos):
"""
Test that when command is run without commit, correct information is logged.
"""
course_videos = [
(self.course.id, 'super-soaker', 'https://www.youtube.com/watch?v=OscRe3pSP80'),
(self.course_2.id, 'medium-soaker', 'https://www.youtube.com/watch?v=OscRe3pSP81')
]
mock_course_videos.return_value = course_videos
setup_video_thumbnails_config(all_course_videos=True)
with LogCapture(LOGGER_NAME, level=logging.INFO) as logger:
call_command('video_thumbnails')
# Verify that list of course video ids is logged.
logger.check(
(
LOGGER_NAME, 'INFO',
('[Video Thumbnails] Videos(total): 2, '
'Videos(updated): 0, Videos(non-updated): 2, '
'Videos(update-in-process): 2')
),
(
LOGGER_NAME, 'INFO',
'[video thumbnails] selected course videos: {course_videos} '.format(
course_videos=text_type(course_videos)
)
)
)
# Verify that `enqueue_update_thumbnail_tasks` is not called.
self.assertFalse(mock_enqueue_thumbnails.called)
@patch('edxval.api.get_course_video_ids_with_youtube_profile')
@patch('contentstore.management.commands.video_thumbnails.enqueue_update_thumbnail_tasks')
def test_video_thumbnails_with_commit(self, mock_enqueue_thumbnails, mock_course_videos):
"""
Test that when command is run with with commit, it works as expected.
"""
course_videos = [
(self.course.id, 'super-soaker', 'https://www.youtube.com/watch?v=OscRe3pSP80'),
(self.course_2.id, 'medium-soaker', 'https://www.youtube.com/watch?v=OscRe3pSP81')
]
mock_course_videos.return_value = course_videos
setup_video_thumbnails_config(commit=True, all_course_videos=True)
with LogCapture(LOGGER_NAME, level=logging.INFO) as logger:
call_command('video_thumbnails')
# Verify that command information correctly logged.
logger.check((
LOGGER_NAME, 'INFO',
('[Video Thumbnails] Videos(total): 2, '
'Videos(updated): 0, Videos(non-updated): 2, '
'Videos(update-in-process): 2')
))
# Verify that `enqueue_update_thumbnail_tasks` is called.
self.assertTrue(mock_enqueue_thumbnails.called)
@patch('edxval.api.get_course_video_ids_with_youtube_profile')
@patch('contentstore.video_utils.download_youtube_video_thumbnail') # Mock(side_effect=Exception())
def test_video_thumbnails_scraping_failed(self, mock_scrape_thumbnails, mock_course_videos):
"""
Test that when scraping fails, it is handled correclty.
"""
course_videos = [
(self.course.id, 'super-soaker', 'OscRe3pSP80'),
(self.course_2.id, 'medium-soaker', 'OscRe3pSP81')
]
mock_scrape_thumbnails.side_effect = Exception('error')
mock_course_videos.return_value = course_videos
setup_video_thumbnails_config(commit=True, all_course_videos=True)
tasks_logger = "cms.djangoapps.contentstore.tasks"
with LogCapture(tasks_logger, level=logging.INFO) as logger:
call_command('video_thumbnails')
# Verify that tasks information is correctly logged.
logger.check(
(
tasks_logger, 'ERROR',
("[video thumbnails] [run=1] [video-thumbnails-scraping-failed-with-unknown-exc] "
"[edx_video_id=super-soaker] [youtube_id=OscRe3pSP80] [course={}]".format(self.course.id))
),
(
tasks_logger, 'ERROR',
("[video thumbnails] [run=1] [video-thumbnails-scraping-failed-with-unknown-exc] "
"[edx_video_id=medium-soaker] [youtube_id=OscRe3pSP81] [course={}]".format(self.course_2.id))
)
)

101
cms/djangoapps/contentstore/management/commands/video_thumbnails.py Normal file
View File

@@ -0,0 +1,101 @@
"""
Command to scrape thumbnails and add them to the course-videos.
"""
import logging
from six import text_type
import edxval.api as edxval_api
from django.core.management import BaseCommand
from django.core.management.base import CommandError
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
from openedx.core.djangoapps.video_config.models import VideoThumbnailSetting, UpdatedCourseVideos
from cms.djangoapps.contentstore.tasks import enqueue_update_thumbnail_tasks
log = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Example usage:
$ ./manage.py cms video_thumbnails
"""
help = 'Adds thumbnails from YouTube to videos'
def _get_command_options(self):
"""
Returns the command arguments configured via django admin.
"""
command_settings = self._latest_settings()
commit = command_settings.commit
if command_settings.all_course_videos:
all_course_videos = edxval_api.get_course_video_ids_with_youtube_profile()
updated_course_videos = UpdatedCourseVideos.objects.all().values_list('course_id', 'edx_video_id')
non_updated_course_videos = [
course_video
for course_video in all_course_videos
if (course_video[0], course_video[1]) not in list(updated_course_videos)
]
# Course videos for whom video thumbnails need to be updated
course_videos = non_updated_course_videos[:command_settings.batch_size]
log.info(
('[Video Thumbnails] Videos(total): %s, '
'Videos(updated): %s, Videos(non-updated): %s, '
'Videos(update-in-process): %s'),
len(all_course_videos),
len(updated_course_videos),
len(non_updated_course_videos),
len(course_videos),
)
else:
validated_course_ids = self._validate_course_ids(command_settings.course_ids.split())
course_videos = edxval_api.get_course_video_ids_with_youtube_profile(validated_course_ids)
return course_videos, commit
def _validate_course_ids(self, course_ids):
"""
Validate a list of course key strings.
"""
try:
for course_id in course_ids:
CourseKey.from_string(course_id)
return course_ids
except InvalidKeyError as error:
raise CommandError('Invalid key specified: {}'.format(text_type(error)))
def _latest_settings(self):
"""
Return the latest version of the VideoThumbnailSetting
"""
return VideoThumbnailSetting.current()
def handle(self, *args, **options):
"""
Invokes the video thumbnail enqueue function.
"""
video_thumbnail_settings = self._latest_settings()
videos_per_task = video_thumbnail_settings.videos_per_task
course_videos, commit = self._get_command_options()
if commit:
command_run = video_thumbnail_settings.increment_run()
enqueue_update_thumbnail_tasks(
course_videos=course_videos,
videos_per_task=videos_per_task,
run=command_run
)
if video_thumbnail_settings.all_course_videos:
for course_id, edx_video_id, __ in course_videos:
UpdatedCourseVideos.objects.get_or_create(
course_id=course_id,
edx_video_id=edx_video_id,
command_run=command_run
)
else:
log.info('[video thumbnails] selected course videos: {course_videos} '.format(
course_videos=text_type(course_videos)
))

80
cms/djangoapps/contentstore/tasks.py
View File

@@ -9,6 +9,7 @@ import os
import shutil
import tarfile
from datetime import datetime
from math import ceil
from tempfile import NamedTemporaryFile, mkdtemp
from celery import group
@@ -39,6 +40,7 @@ import dogstats_wrapper as dog_stats_api
from contentstore.courseware_index import CoursewareSearchIndexer, LibrarySearchIndexer, SearchIndexingError
from contentstore.storage import course_import_export_storage
from contentstore.utils import initialize_permissions, reverse_usage_url
from contentstore.video_utils import scrape_youtube_thumbnail
from course_action_state.models import CourseRerunState
from models.settings.course_metadata import CourseMetadata
from openedx.core.djangoapps.embargo.models import CountryAccessRule, RestrictedCourse
@@ -84,6 +86,84 @@ COURSE_LEVEL_TIMEOUT_SECONDS = 1200
VIDEO_LEVEL_TIMEOUT_SECONDS = 300
def enqueue_update_thumbnail_tasks(course_videos, videos_per_task, run):
"""
Enqueue tasks to update video thumbnails from youtube.
Arguments:
course_videos: A list of tuples, each containing course ID, video ID and youtube ID.
videos_per_task: Number of course videos that can be processed by a single celery task.
run: This tracks the YT thumbnail scraping job runs.
"""
tasks = []
batch_size = len(course_videos)
# Further slice the course-videos batch into chunks on the
# basis of number of course-videos per task.
start = 0
end = videos_per_task
chunks_count = int(ceil(batch_size / float(videos_per_task)))
for __ in xrange(0, chunks_count):
course_videos_chunk = course_videos[start:end]
tasks.append(task_scrape_youtube_thumbnail.s(
course_videos_chunk, run
))
start = end
end += videos_per_task
# Kick off a chord of scraping tasks
callback = task_scrape_youtube_thumbnail_callback.s(
run=run,
batch_size=batch_size,
videos_per_task=videos_per_task,
)
chord(tasks)(callback)
@chord_task(bind=True, routing_key=settings.VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE)
def task_scrape_youtube_thumbnail_callback(self, results, run, # pylint: disable=unused-argument
batch_size, videos_per_task):
"""
Callback for collating the results of yt thumbnails scraping tasks chord.
"""
yt_thumbnails_scraping_tasks_count = len(list(results()))
LOGGER.info(
("[video thumbnails] [run=%s] [video-thumbnails-scraping-complete-for-a-batch] [tasks_count=%s] "
"[batch_size=%s] [videos_per_task=%s]"),
run, yt_thumbnails_scraping_tasks_count, batch_size, videos_per_task
)
@chord_task(
bind=True,
base=LoggedPersistOnFailureTask,
default_retry_delay=RETRY_DELAY_SECONDS,
max_retries=1,
time_limit=COURSE_LEVEL_TIMEOUT_SECONDS,
routing_key=settings.SCRAPE_YOUTUBE_THUMBNAILS_JOB_QUEUE
)
def task_scrape_youtube_thumbnail(self, course_videos, run): # pylint: disable=unused-argument
"""
Task to scrape youtube thumbnails and update them in edxval for the given course-videos.
Arguments:
course_videos: A list of tuples, each containing course ID, video ID and youtube ID.
run: This tracks the YT thumbnail scraping job runs.
"""
for course_id, edx_video_id, youtube_id in course_videos:
try:
scrape_youtube_thumbnail(course_id, edx_video_id, youtube_id)
except Exception: # pylint: disable=broad-except
LOGGER.exception(
("[video thumbnails] [run=%s] [video-thumbnails-scraping-failed-with-unknown-exc] "
"[edx_video_id=%s] [youtube_id=%s] [course=%s]"),
run,
edx_video_id,
youtube_id,
course_id
)
continue
@chord_task(bind=True, routing_key=settings.VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE)
def task_status_callback(self, results, revision, # pylint: disable=unused-argument
course_id, command_run, video_location):

4
cms/djangoapps/contentstore/video_utils.py
View File

@@ -85,7 +85,7 @@ def download_youtube_video_thumbnail(youtube_id):
Download highest resoultion video thumbnail available from youtube.
"""
thumbnail_content = thumbnail_content_type = None
# Download highest resoultion thumbnail available.
# Download highest resolution thumbnail available.
for thumbnail_quality in YOUTUBE_THUMBNAIL_SIZES:
thumbnail_url = urljoin('https://img.youtube.com', '/vi/{youtube_id}/{thumbnail_quality}.jpg'.format(
youtube_id=youtube_id, thumbnail_quality=thumbnail_quality
@@ -94,7 +94,7 @@ def download_youtube_video_thumbnail(youtube_id):
if response.status_code == requests.codes.ok: # pylint: disable=no-member
thumbnail_content = response.content
thumbnail_content_type = response.headers['content-type']
# If best available resolution is found, skip looking for lower resolutions.
# If best available resolution is found, don't look for lower resolutions.
break
return thumbnail_content, thumbnail_content_type

3
cms/envs/aws.py
View File

@@ -544,6 +544,9 @@ COURSEGRAPH_JOB_QUEUE = ENV_TOKENS.get('COURSEGRAPH_JOB_QUEUE', LOW_PRIORITY_QUE
########## Settings for video transcript migration tasks ############
VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE = ENV_TOKENS.get('VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE', LOW_PRIORITY_QUEUE)
########## Settings youtube thumbnails scraper tasks ############
SCRAPE_YOUTUBE_THUMBNAILS_JOB_QUEUE = ENV_TOKENS.get('SCRAPE_YOUTUBE_THUMBNAILS_JOB_QUEUE', LOW_PRIORITY_QUEUE)
########################## Parental controls config #######################
# The age at which a learner no longer requires parental consent, or None

3
cms/envs/common.py
View File

@@ -1511,6 +1511,9 @@ COURSEGRAPH_JOB_QUEUE = LOW_PRIORITY_QUEUE
########## Settings for video transcript migration tasks ############
VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE = LOW_PRIORITY_QUEUE
########## Settings youtube thumbnails scraper tasks ############
SCRAPE_YOUTUBE_THUMBNAILS_JOB_QUEUE = LOW_PRIORITY_QUEUE
###################### VIDEO IMAGE STORAGE ######################
VIDEO_IMAGE_DEFAULT_FILENAME = 'images/video-images/default_video_image.png'

16
openedx/core/djangoapps/video_config/admin.py
View File

@@ -13,6 +13,7 @@ from openedx.core.djangoapps.video_config.models import (
CourseHLSPlaybackEnabledFlag, HLSPlaybackEnabledFlag,
CourseVideoTranscriptEnabledFlag, VideoTranscriptEnabledFlag,
TranscriptMigrationSetting, MigrationEnqueuedCourse,
VideoThumbnailSetting, UpdatedCourseVideos,
)
@@ -62,6 +63,19 @@ class MigrationEnqueuedCourseAdmin(admin.ModelAdmin):
search_fields = ['course_id', 'command_run']
class UpdatedCourseVideosAdmin(admin.ModelAdmin):
"""
Read-only list/search view of the videos whose thumbnails have been updated.
"""
list_display = [
'course_id',
'edx_video_id',
'command_run',
]
search_fields = ['course_id', 'edx_video_id', 'command_run']
admin.site.register(HLSPlaybackEnabledFlag, ConfigurationModelAdmin)
admin.site.register(CourseHLSPlaybackEnabledFlag, CourseHLSPlaybackEnabledFlagAdmin)
@@ -69,3 +83,5 @@ admin.site.register(VideoTranscriptEnabledFlag, ConfigurationModelAdmin)
admin.site.register(CourseVideoTranscriptEnabledFlag, CourseVideoTranscriptEnabledFlagAdmin)
admin.site.register(TranscriptMigrationSetting, ConfigurationModelAdmin)
admin.site.register(MigrationEnqueuedCourse, MigrationEnqueuedCourseAdmin)
admin.site.register(VideoThumbnailSetting, ConfigurationModelAdmin)
admin.site.register(UpdatedCourseVideos, UpdatedCourseVideosAdmin)

55
openedx/core/djangoapps/video_config/migrations/0006_videothumbnailetting_updatedcoursevideos.py Normal file
View File

@@ -0,0 +1,55 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.15 on 2018-08-15 19:13
from __future__ import unicode_literals
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
import model_utils.fields
import opaque_keys.edx.django.models
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('video_config', '0005_auto_20180719_0752'),
]
operations = [
migrations.CreateModel(
name='UpdatedCourseVideos',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
('course_id', opaque_keys.edx.django.models.CourseKeyField(db_index=True, max_length=255)),
('edx_video_id', models.CharField(max_length=100)),
('command_run', models.PositiveIntegerField(default=0)),
],
),
migrations.CreateModel(
name='VideoThumbnailSetting',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('change_date', models.DateTimeField(auto_now_add=True, verbose_name='Change date')),
('enabled', models.BooleanField(default=False, verbose_name='Enabled')),
('command_run', models.PositiveIntegerField(default=0)),
('batch_size', models.PositiveIntegerField(default=0)),
('videos_per_task', models.PositiveIntegerField(default=0)),
('commit', models.BooleanField(default=False, help_text=b'Dry-run or commit.')),
('all_course_videos', models.BooleanField(default=False, help_text=b'Process all videos.')),
('course_ids', models.TextField(blank=True, help_text=b'Whitespace-separated list of course ids for which to update videos.')),
('changed_by', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL, verbose_name='Changed by')),
],
options={
'ordering': ('-change_date',),
'abstract': False,
},
),
migrations.AlterUniqueTogether(
name='updatedcoursevideos',
unique_together=set([('course_id', 'edx_video_id')]),
),
]

57
openedx/core/djangoapps/video_config/models.py
View File

@@ -1,13 +1,16 @@
"""
Configuration models for Video XModule
"""
from django.db import models
from django.db.models import BooleanField, TextField, PositiveIntegerField
from config_models.models import ConfigurationModel
from model_utils.models import TimeStampedModel
from opaque_keys.edx.django.models import CourseKeyField
URL_REGEX = r'^[a-zA-Z0-9\-_]*$'
class HLSPlaybackEnabledFlag(ConfigurationModel):
"""
Enables HLS Playback across the platform.
@@ -186,3 +189,55 @@ class MigrationEnqueuedCourse(TimeStampedModel):
return u'MigrationEnqueuedCourse: ID={course_id}, Run={command_run}'.format(
course_id=self.course_id, command_run=self.command_run
)
class VideoThumbnailSetting(ConfigurationModel):
"""
Arguments for the Video Thumbnail management command
"""
command_run = PositiveIntegerField(default=0)
batch_size = PositiveIntegerField(default=0)
videos_per_task = PositiveIntegerField(default=0)
commit = BooleanField(
default=False,
help_text="Dry-run or commit."
)
all_course_videos = BooleanField(
default=False,
help_text="Process all videos."
)
course_ids = TextField(
blank=True,
help_text="Whitespace-separated list of course ids for which to update videos."
)
def increment_run(self):
"""
Increments the run which indicates the management command run count.
"""
self.command_run += 1
self.save()
return self.command_run
def __unicode__(self):
return "[VideoThumbnailSetting] update for {courses} courses if commit as {commit}".format(
courses='ALL' if self.all_course_videos else self.course_ids,
commit=self.commit,
)
class UpdatedCourseVideos(TimeStampedModel):
"""
Temporary model to persist the course videos which have been enqueued to update video thumbnails.
"""
course_id = CourseKeyField(db_index=True, max_length=255)
edx_video_id = models.CharField(max_length=100)
command_run = PositiveIntegerField(default=0)
class Meta:
unique_together = ('course_id', 'edx_video_id')
def __unicode__(self):
return u'UpdatedCourseVideos: CourseID={course_id}, VideoID={video_id}, Run={command_run}'.format(
course_id=self.course_id, video_id=self.edx_video_id, command_run=self.command_run
)

3
requirements/edx/base.txt
View File

@@ -128,7 +128,8 @@ edx-rest-api-client==1.8.2
edx-search==1.2.1
edx-submissions==2.0.12
edx-user-state-client==1.0.4
edxval==0.1.18
#edxval==0.1.18
-e git+https://github.com/edx/edx-val.git@iahmad/EDUCATOR-3109-Get-all-videos#egg=edxval
elasticsearch==1.9.0 # via edx-search
enum34==1.1.6
event-tracking==0.2.4

3
requirements/edx/development.txt
View File

@@ -149,7 +149,8 @@ edx-search==1.2.1
edx-sphinx-theme==1.3.0
edx-submissions==2.0.12
edx-user-state-client==1.0.4
edxval==0.1.18
#edxval==0.1.18
-e git+https://github.com/edx/edx-val.git@iahmad/EDUCATOR-3109-Get-all-videos#egg=edxval
elasticsearch==1.9.0
enum34==1.1.6
event-tracking==0.2.4

3
requirements/edx/testing.txt
View File

@@ -143,7 +143,8 @@ edx-rest-api-client==1.8.2
edx-search==1.2.1
edx-submissions==2.0.12
edx-user-state-client==1.0.4
edxval==0.1.18
#edxval==0.1.18
-e git+https://github.com/edx/edx-val.git@iahmad/EDUCATOR-3109-Get-all-videos#egg=edxval
elasticsearch==1.9.0
enum34==1.1.6
event-tracking==0.2.4