diff --git a/cms/djangoapps/contentstore/management/commands/tests/test_video_thumbnails.py b/cms/djangoapps/contentstore/management/commands/tests/test_video_thumbnails.py index 800646c24a..afb23b47db 100644 --- a/cms/djangoapps/contentstore/management/commands/tests/test_video_thumbnails.py +++ b/cms/djangoapps/contentstore/management/commands/tests/test_video_thumbnails.py @@ -66,9 +66,7 @@ class TestVideoThumbnails(ModuleStoreTestCase): logger.check( ( LOGGER_NAME, 'INFO', - ('[Video Thumbnails] Videos(total): 2, ' - 'Videos(updated): 0, Videos(non-updated): 2, ' - 'Videos(update-in-process): 2') + '[Video Thumbnails] Videos(updated): 0, Videos(update-in-process): 2' ), ( LOGGER_NAME, 'INFO', @@ -98,9 +96,7 @@ class TestVideoThumbnails(ModuleStoreTestCase): # Verify that command information correctly logged. logger.check(( LOGGER_NAME, 'INFO', - ('[Video Thumbnails] Videos(total): 2, ' - 'Videos(updated): 0, Videos(non-updated): 2, ' - 'Videos(update-in-process): 2') + '[Video Thumbnails] Videos(updated): 0, Videos(update-in-process): 2' )) # Verify that `enqueue_update_thumbnail_tasks` is called. self.assertTrue(mock_enqueue_thumbnails.called) diff --git a/cms/djangoapps/contentstore/management/commands/video_thumbnails.py b/cms/djangoapps/contentstore/management/commands/video_thumbnails.py index 8a4b17f655..7341f437ef 100644 --- a/cms/djangoapps/contentstore/management/commands/video_thumbnails.py +++ b/cms/djangoapps/contentstore/management/commands/video_thumbnails.py @@ -10,7 +10,7 @@ from django.core.management.base import CommandError from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey -from openedx.core.djangoapps.video_config.models import VideoThumbnailSetting, UpdatedCourseVideos +from openedx.core.djangoapps.video_config.models import VideoThumbnailSetting from cms.djangoapps.contentstore.tasks import enqueue_update_thumbnail_tasks log = logging.getLogger(__name__) @@ -30,24 +30,12 @@ class Command(BaseCommand): command_settings = self._latest_settings() commit = command_settings.commit if command_settings.all_course_videos: - all_course_videos = edxval_api.get_course_video_ids_with_youtube_profile() - updated_course_videos = UpdatedCourseVideos.objects.all().values_list('course_id', 'edx_video_id') - non_updated_course_videos = [ - course_video - for course_video in all_course_videos - if (course_video[0], course_video[1]) not in list(updated_course_videos) - ] - # Course videos for whom video thumbnails need to be updated - course_videos = non_updated_course_videos[:command_settings.batch_size] - + course_videos = edxval_api.get_course_video_ids_with_youtube_profile( + offset=command_settings.offset, limit=command_settings.batch_size + ) log.info( - ('[Video Thumbnails] Videos(total): %s, ' - 'Videos(updated): %s, Videos(non-updated): %s, ' - 'Videos(update-in-process): %s'), - len(all_course_videos), - len(updated_course_videos), - len(non_updated_course_videos), - len(course_videos), + '[Video Thumbnails] Videos(updated): %s, Videos(update-in-process): %s', + command_settings.offset, len(course_videos), ) else: validated_course_ids = self._validate_course_ids(command_settings.course_ids.split()) @@ -89,12 +77,7 @@ class Command(BaseCommand): run=command_run ) if video_thumbnail_settings.all_course_videos: - for course_id, edx_video_id, __ in course_videos: - UpdatedCourseVideos.objects.get_or_create( - course_id=course_id, - edx_video_id=edx_video_id, - command_run=command_run - ) + video_thumbnail_settings.update_offset() else: log.info('[video thumbnails] selected course videos: {course_videos} '.format( course_videos=text_type(course_videos) diff --git a/openedx/core/djangoapps/video_config/migrations/0007_videothumbnailsetting_offset.py b/openedx/core/djangoapps/video_config/migrations/0007_videothumbnailsetting_offset.py new file mode 100644 index 0000000000..6d534cd429 --- /dev/null +++ b/openedx/core/djangoapps/video_config/migrations/0007_videothumbnailsetting_offset.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.15 on 2018-09-25 13:41 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('video_config', '0006_videothumbnailetting_updatedcoursevideos'), + ] + + operations = [ + migrations.AddField( + model_name='videothumbnailsetting', + name='offset', + field=models.PositiveIntegerField(default=0), + ), + ] diff --git a/openedx/core/djangoapps/video_config/models.py b/openedx/core/djangoapps/video_config/models.py index 7da3ce421f..96c7ee696b 100644 --- a/openedx/core/djangoapps/video_config/models.py +++ b/openedx/core/djangoapps/video_config/models.py @@ -196,6 +196,7 @@ class VideoThumbnailSetting(ConfigurationModel): Arguments for the Video Thumbnail management command """ command_run = PositiveIntegerField(default=0) + offset = PositiveIntegerField(default=0) batch_size = PositiveIntegerField(default=0) videos_per_task = PositiveIntegerField(default=0) commit = BooleanField( @@ -219,6 +220,10 @@ class VideoThumbnailSetting(ConfigurationModel): self.save() return self.command_run + def update_offset(self): + self.offset += self.batch_size + self.save() + def __unicode__(self): return "[VideoThumbnailSetting] update for {courses} courses if commit as {commit}".format( courses='ALL' if self.all_course_videos else self.course_ids, diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 29277b6a86..ba133c4977 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -129,7 +129,7 @@ edx-rest-api-client==1.8.2 edx-search==1.2.1 edx-submissions==2.0.12 edx-user-state-client==1.0.4 -edxval==0.1.20 +edxval==0.1.21 elasticsearch==1.9.0 # via edx-search enum34==1.1.6 event-tracking==0.2.4 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 1c0379ae33..ed4dd027d7 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -150,7 +150,7 @@ edx-search==1.2.1 edx-sphinx-theme==1.3.0 edx-submissions==2.0.12 edx-user-state-client==1.0.4 -edxval==0.1.20 +edxval==0.1.21 elasticsearch==1.9.0 enum34==1.1.6 event-tracking==0.2.4 diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index a1e82cd8df..45565adfa7 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -144,7 +144,7 @@ edx-rest-api-client==1.8.2 edx-search==1.2.1 edx-submissions==2.0.12 edx-user-state-client==1.0.4 -edxval==0.1.20 +edxval==0.1.21 elasticsearch==1.9.0 enum34==1.1.6 event-tracking==0.2.4