Update VAL version and thumbnails scrapper job in accordance with val api

cms/djangoapps/contentstore/management/commands/tests/test_video_thumbnails.py

@@ -66,9 +66,7 @@ class TestVideoThumbnails(ModuleStoreTestCase):
             logger.check(
                 (
                     LOGGER_NAME, 'INFO',
-                    ('[Video Thumbnails] Videos(total): 2, '
-                     'Videos(updated): 0, Videos(non-updated): 2, '
-                     'Videos(update-in-process): 2')
+                    '[Video Thumbnails] Videos(updated): 0, Videos(update-in-process): 2'
                 ),
                 (
                     LOGGER_NAME, 'INFO',
@@ -98,9 +96,7 @@ class TestVideoThumbnails(ModuleStoreTestCase):
             # Verify that command information correctly logged.
             logger.check((
                 LOGGER_NAME, 'INFO',
-                ('[Video Thumbnails] Videos(total): 2, '
-                 'Videos(updated): 0, Videos(non-updated): 2, '
-                 'Videos(update-in-process): 2')
+                '[Video Thumbnails] Videos(updated): 0, Videos(update-in-process): 2'
             ))
             # Verify that `enqueue_update_thumbnail_tasks` is called.
             self.assertTrue(mock_enqueue_thumbnails.called)

cms/djangoapps/contentstore/management/commands/video_thumbnails.py

@@ -10,7 +10,7 @@ from django.core.management.base import CommandError
 from opaque_keys import InvalidKeyError
 from opaque_keys.edx.keys import CourseKey
 
-from openedx.core.djangoapps.video_config.models import VideoThumbnailSetting, UpdatedCourseVideos
+from openedx.core.djangoapps.video_config.models import VideoThumbnailSetting
 from cms.djangoapps.contentstore.tasks import enqueue_update_thumbnail_tasks
 
 log = logging.getLogger(__name__)
@@ -30,24 +30,12 @@ class Command(BaseCommand):
         command_settings = self._latest_settings()
         commit = command_settings.commit
         if command_settings.all_course_videos:
-            all_course_videos = edxval_api.get_course_video_ids_with_youtube_profile()
-            updated_course_videos = UpdatedCourseVideos.objects.all().values_list('course_id', 'edx_video_id')
-            non_updated_course_videos = [
-                course_video
-                for course_video in all_course_videos
-                if (course_video[0], course_video[1]) not in list(updated_course_videos)
-            ]
-            # Course videos for whom video thumbnails need to be updated
-            course_videos = non_updated_course_videos[:command_settings.batch_size]
-
+            course_videos = edxval_api.get_course_video_ids_with_youtube_profile(
+                offset=command_settings.offset, limit=command_settings.batch_size
+            )
             log.info(
-                ('[Video Thumbnails] Videos(total): %s, '
-                 'Videos(updated): %s, Videos(non-updated): %s, '
-                 'Videos(update-in-process): %s'),
-                len(all_course_videos),
-                len(updated_course_videos),
-                len(non_updated_course_videos),
-                len(course_videos),
+                '[Video Thumbnails] Videos(updated): %s, Videos(update-in-process): %s',
+                command_settings.offset, len(course_videos),
             )
         else:
             validated_course_ids = self._validate_course_ids(command_settings.course_ids.split())
@@ -89,12 +77,7 @@ class Command(BaseCommand):
                 run=command_run
             )
             if video_thumbnail_settings.all_course_videos:
-                for course_id, edx_video_id, __ in course_videos:
-                    UpdatedCourseVideos.objects.get_or_create(
-                        course_id=course_id,
-                        edx_video_id=edx_video_id,
-                        command_run=command_run
-                    )
+                video_thumbnail_settings.update_offset()
         else:
             log.info('[video thumbnails] selected course videos: {course_videos} '.format(
                 course_videos=text_type(course_videos)

openedx/core/djangoapps/video_config/migrations/0007_videothumbnailsetting_offset.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.15 on 2018-09-25 13:41
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('video_config', '0006_videothumbnailetting_updatedcoursevideos'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='videothumbnailsetting',
+            name='offset',
+            field=models.PositiveIntegerField(default=0),
+        ),
+    ]

openedx/core/djangoapps/video_config/models.py

@@ -196,6 +196,7 @@ class VideoThumbnailSetting(ConfigurationModel):
     Arguments for the Video Thumbnail management command
     """
     command_run = PositiveIntegerField(default=0)
+    offset = PositiveIntegerField(default=0)
     batch_size = PositiveIntegerField(default=0)
     videos_per_task = PositiveIntegerField(default=0)
     commit = BooleanField(
@@ -219,6 +220,10 @@ class VideoThumbnailSetting(ConfigurationModel):
         self.save()
         return self.command_run
 
+    def update_offset(self):
+        self.offset += self.batch_size
+        self.save()
+
     def __unicode__(self):
         return "[VideoThumbnailSetting] update for {courses} courses if commit as {commit}".format(
             courses='ALL' if self.all_course_videos else self.course_ids,

requirements/edx/base.txt

@@ -129,7 +129,7 @@ edx-rest-api-client==1.8.2
 edx-search==1.2.1
 edx-submissions==2.0.12
 edx-user-state-client==1.0.4
-edxval==0.1.20
+edxval==0.1.21
 elasticsearch==1.9.0      # via edx-search
 enum34==1.1.6
 event-tracking==0.2.4

requirements/edx/development.txt

@@ -150,7 +150,7 @@ edx-search==1.2.1
 edx-sphinx-theme==1.3.0
 edx-submissions==2.0.12
 edx-user-state-client==1.0.4
-edxval==0.1.20
+edxval==0.1.21
 elasticsearch==1.9.0
 enum34==1.1.6
 event-tracking==0.2.4

requirements/edx/testing.txt

@@ -144,7 +144,7 @@ edx-rest-api-client==1.8.2
 edx-search==1.2.1
 edx-submissions==2.0.12
 edx-user-state-client==1.0.4
-edxval==0.1.20
+edxval==0.1.21
 elasticsearch==1.9.0
 enum34==1.1.6
 event-tracking==0.2.4