Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Remove django-celery from transitive deps; remove unused tasks (#23318)

Browse Source 
The video thumbnail and transcript tasks were the only things using chord_task from
edx-celeryutils, which in turn was blocking django-celery removal. But, they're no longer used.

Co-authored-by: Diana Huang <diana.k.huang@gmail.com>
This commit is contained in:
Tim McCormack
2020-03-05 21:57:12 +00:00
committed by GitHub
parent 919dc5760a
commit 11fc118402
8 changed files with 19 additions and 1071 deletions
Download Patch File Download Diff File Expand all files Collapse all files

155
cms/djangoapps/contentstore/management/commands/migrate_transcripts.py
View File

@@ -1,155 +0,0 @@
"""
Command to migrate transcripts to django storage.
"""
import logging
from django.core.management import BaseCommand, CommandError
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import CourseLocator
from six.moves import map
from cms.djangoapps.contentstore.tasks import (
DEFAULT_ALL_COURSES,
DEFAULT_COMMIT,
DEFAULT_FORCE_UPDATE,
enqueue_async_migrate_transcripts_tasks
)
from openedx.core.djangoapps.video_config.models import MigrationEnqueuedCourse, TranscriptMigrationSetting
from openedx.core.lib.command_utils import get_mutually_exclusive_required_option, parse_course_keys
from xmodule.modulestore.django import modulestore
log = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Example usage:
$ ./manage.py cms migrate_transcripts --all-courses --force-update --commit
$ ./manage.py cms migrate_transcripts --course-id 'Course1' --course-id 'Course2' --commit
$ ./manage.py cms migrate_transcripts --from-settings
"""
help = 'Migrates transcripts to S3 for one or more courses.'
def add_arguments(self, parser):
"""
Add arguments to the command parser.
"""
parser.add_argument(
'--course-id', '--course_id',
dest='course_ids',
action='append',
help=u'Migrates transcripts for the list of courses.'
)
parser.add_argument(
'--all-courses', '--all', '--all_courses',
dest='all_courses',
action='store_true',
default=DEFAULT_ALL_COURSES,
help=u'Migrates transcripts to the configured django storage for all courses.'
)
parser.add_argument(
'--from-settings', '--from_settings',
dest='from_settings',
help='Migrate Transcripts with settings set via django admin',
action='store_true',
default=False,
)
parser.add_argument(
'--force-update', '--force_update',
dest='force_update',
action='store_true',
default=DEFAULT_FORCE_UPDATE,
help=u'Force migrate transcripts for the requested courses, overwrite if already present.'
)
parser.add_argument(
'--commit',
dest='commit',
action='store_true',
default=DEFAULT_COMMIT,
help=u'Commits the discovered video transcripts to django storage. '
u'Without this flag, the command will return the transcripts discovered for migration.'
)
def _parse_course_key(self, raw_value):
""" Parses course key from string """
try:
result = CourseKey.from_string(raw_value)
except InvalidKeyError:
raise CommandError(u"Invalid course_key: '%s'." % raw_value)
if not isinstance(result, CourseLocator):
raise CommandError(u"Argument {0} is not a course key".format(raw_value))
return result
def _get_migration_options(self, options):
"""
Returns the command arguments configured via django admin.
"""
force_update = options['force_update']
commit = options['commit']
courses_mode = get_mutually_exclusive_required_option(options, 'course_ids', 'all_courses', 'from_settings')
if courses_mode == 'all_courses':
course_keys = [course.id for course in modulestore().get_course_summaries()]
elif courses_mode == 'course_ids':
course_keys = list(map(self._parse_course_key, options['course_ids']))
else:
migration_settings = self._latest_settings()
if migration_settings.all_courses:
all_courses = [course.id for course in modulestore().get_course_summaries()]
# Following is to avoid re-rerunning migrations for the already enqueued courses.
# Although the migrations job is idempotent, but we need to track if the transcript migration
# job was initiated for specific course(s) in order to elevate load from the workers and for
# the job to be able identify the past enqueued courses.
migrated_courses = MigrationEnqueuedCourse.objects.all().values_list('course_id', flat=True)
non_migrated_courses = [
course_key
for course_key in all_courses
if course_key not in migrated_courses
]
# Course batch to be migrated.
course_keys = non_migrated_courses[:migration_settings.batch_size]
log.info(
(u'[Transcript Migration] Courses(total): %s, '
u'Courses(migrated): %s, Courses(non-migrated): %s, '
u'Courses(migration-in-process): %s'),
len(all_courses),
len(migrated_courses),
len(non_migrated_courses),
len(course_keys),
)
else:
course_keys = parse_course_keys(migration_settings.course_ids.split())
force_update = migration_settings.force_update
commit = migration_settings.commit
return course_keys, force_update, commit
def _latest_settings(self):
"""
Return the latest version of the TranscriptMigrationSetting
"""
return TranscriptMigrationSetting.current()
def handle(self, *args, **options):
"""
Invokes the migrate transcripts enqueue function.
"""
migration_settings = self._latest_settings()
course_keys, force_update, commit = self._get_migration_options(options)
command_run = migration_settings.increment_run() if commit else -1
enqueue_async_migrate_transcripts_tasks(
course_keys=course_keys, commit=commit, command_run=command_run, force_update=force_update
)
if commit and options.get('from_settings') and migration_settings.all_courses:
for course_key in course_keys:
enqueued_course, created = MigrationEnqueuedCourse.objects.get_or_create(course_id=course_key)
if created:
enqueued_course.command_run = command_run
enqueued_course.save()

332
cms/djangoapps/contentstore/management/commands/tests/test_migrate_transcripts.py
View File

@@ -1,332 +0,0 @@
# -*- coding: utf-8 -*-
"""
Tests for course transcript migration management command.
"""
import itertools
import logging
from datetime import datetime
import ddt
import pytz
import six
from django.core.management import CommandError, call_command
from django.test import TestCase
from edxval import api as api
from mock import patch
from testfixtures import LogCapture
from openedx.core.djangoapps.video_config.models import MigrationEnqueuedCourse, TranscriptMigrationSetting
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from xmodule.video_module import VideoBlock
from xmodule.video_module.transcripts_utils import save_to_store
LOGGER_NAME = "cms.djangoapps.contentstore.tasks"
SRT_FILEDATA = '''
0
00:00:00,270 --> 00:00:02,720
sprechen sie deutsch?
1
00:00:02,720 --> 00:00:05,430
Ja, ich spreche Deutsch
2
00:00:6,500 --> 00:00:08,600
可以用“我不太懂艺术 但我知道我喜欢什么”做比喻
'''
CRO_SRT_FILEDATA = '''
0
00:00:00,270 --> 00:00:02,720
Dobar dan!
1
00:00:02,720 --> 00:00:05,430
Kako ste danas?
2
00:00:6,500 --> 00:00:08,600
可以用“我不太懂艺术 但我知道我喜欢什么”做比喻
'''
VIDEO_DICT_STAR = dict(
client_video_id='TWINKLE TWINKLE',
duration=42.0,
edx_video_id='test_edx_video_id',
status='upload',
)
class TestArgParsing(TestCase):
"""
Tests for parsing arguments for the `migrate_transcripts` management command
"""
def test_no_args(self):
errstring = "Must specify exactly one of --course_ids, --all_courses, --from_settings"
with self.assertRaisesRegex(CommandError, errstring):
call_command('migrate_transcripts')
def test_invalid_course(self):
errstring = "Invalid course_key: 'invalid-course'."
with self.assertRaisesRegex(CommandError, errstring):
call_command('migrate_transcripts', '--course-id', 'invalid-course')
@ddt.ddt
class TestMigrateTranscripts(ModuleStoreTestCase):
"""
Tests migrating video transcripts in courses from contentstore to django storage
"""
def setUp(self):
""" Common setup. """
super(TestMigrateTranscripts, self).setUp()
self.store = modulestore()
self.course = CourseFactory.create()
self.course_2 = CourseFactory.create()
video = {
'edx_video_id': 'test_edx_video_id',
'client_video_id': 'test1.mp4',
'duration': 42.0,
'status': 'upload',
'courses': [six.text_type(self.course.id)],
'encoded_videos': [],
'created': datetime.now(pytz.utc)
}
api.create_video(video)
video_sample_xml = '''
<video display_name="Test Video"
edx_video_id="test_edx_video_id"
youtube="1.0:p2Q6BrNhdh8,0.75:izygArpw-Qo,1.25:1EeWXzPdhSA,1.5:rABDYkeK0x8"
show_captions="false"
download_track="false"
start_time="1.0"
download_video="false"
end_time="60.0">
<source src="http://www.example.com/source.mp4"/>
<track src="http://www.example.com/track"/>
<handout src="http://www.example.com/handout"/>
<transcript language="ge" src="subs_grmtran1.srt" />
<transcript language="hr" src="subs_croatian1.srt" />
</video>
'''
video_sample_xml_2 = '''
<video display_name="Test Video 2"
edx_video_id="test_edx_video_id_2"
youtube="1.0:p2Q6BrNhdh8,0.75:izygArpw-Qo,1.25:1EeWXzPdhSA,1.5:rABDYkeK0x8"
show_captions="false"
download_track="false"
start_time="1.0"
download_video="false"
end_time="60.0">
<source src="http://www.example.com/source.mp4"/>
<track src="http://www.example.com/track"/>
<handout src="http://www.example.com/handout"/>
<transcript language="ge" src="not_found.srt" />
</video>
'''
self.video_descriptor = ItemFactory.create(
parent_location=self.course.location, category='video',
**VideoBlock.parse_video_xml(video_sample_xml)
)
self.video_descriptor_2 = ItemFactory.create(
parent_location=self.course_2.location, category='video',
**VideoBlock.parse_video_xml(video_sample_xml_2)
)
save_to_store(SRT_FILEDATA, 'subs_grmtran1.srt', 'text/srt', self.video_descriptor.location)
save_to_store(CRO_SRT_FILEDATA, 'subs_croatian1.srt', 'text/srt', self.video_descriptor.location)
def test_migrated_transcripts_count_with_commit(self):
"""
Test migrating transcripts with commit
"""
# check that transcript does not exist
languages = api.get_available_transcript_languages(self.video_descriptor.edx_video_id)
self.assertEqual(len(languages), 0)
self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
# now call migrate_transcripts command and check the transcript availability
call_command('migrate_transcripts', '--course-id', six.text_type(self.course.id), '--commit')
languages = api.get_available_transcript_languages(self.video_descriptor.edx_video_id)
self.assertEqual(len(languages), 2)
self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
def test_migrated_transcripts_without_commit(self):
"""
Test migrating transcripts as a dry-run
"""
# check that transcripts do not exist
languages = api.get_available_transcript_languages(self.video_descriptor.edx_video_id)
self.assertEqual(len(languages), 0)
self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
# now call migrate_transcripts command and check the transcript availability
call_command('migrate_transcripts', '--course-id', six.text_type(self.course.id))
# check that transcripts still do not exist
languages = api.get_available_transcript_languages(self.video_descriptor.edx_video_id)
self.assertEqual(len(languages), 0)
self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
def test_migrate_transcripts_availability(self):
"""
Test migrating transcripts
"""
translations = self.video_descriptor.available_translations(self.video_descriptor.get_transcripts_info())
six.assertCountEqual(self, translations, ['hr', 'ge'])
self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
# now call migrate_transcripts command and check the transcript availability
call_command('migrate_transcripts', '--course-id', six.text_type(self.course.id), '--commit')
self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
def test_migrate_transcripts_idempotency(self):
"""
Test migrating transcripts multiple times
"""
translations = self.video_descriptor.available_translations(self.video_descriptor.get_transcripts_info())
six.assertCountEqual(self, translations, ['hr', 'ge'])
self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
self.assertFalse(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
# now call migrate_transcripts command and check the transcript availability
call_command('migrate_transcripts', '--course-id', six.text_type(self.course.id), '--commit')
self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
# now call migrate_transcripts command again and check the transcript availability
call_command('migrate_transcripts', '--course-id', six.text_type(self.course.id), '--commit')
self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
# now call migrate_transcripts command with --force-update and check the transcript availability
call_command('migrate_transcripts', '--course-id', six.text_type(self.course.id), '--force-update', '--commit')
self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'hr'))
self.assertTrue(api.is_transcript_available(self.video_descriptor.edx_video_id, 'ge'))
def test_migrate_transcripts_logging(self):
"""
Test migrate transcripts logging and output
"""
course_id = six.text_type(self.course.id)
expected_log = (
(
'cms.djangoapps.contentstore.tasks', 'INFO',
(u'[Transcript Migration] [run=-1] [video-transcripts-migration-process-started-for-course] '
u'[course={}]'.format(course_id))
),
(
'cms.djangoapps.contentstore.tasks', 'INFO',
(u'[Transcript Migration] [run=-1] [video-transcript-will-be-migrated] '
u'[revision=rev-opt-published-only] [video={}] [edx_video_id=test_edx_video_id] '
u'[language_code=hr]'.format(self.video_descriptor.location))
),
(
'cms.djangoapps.contentstore.tasks', 'INFO',
(u'[Transcript Migration] [run=-1] [video-transcript-will-be-migrated] '
u'[revision=rev-opt-published-only] [video={}] [edx_video_id=test_edx_video_id] '
u'[language_code=ge]'.format(self.video_descriptor.location))
),
(
'cms.djangoapps.contentstore.tasks', 'INFO',
(u'[Transcript Migration] [run=-1] [transcripts-migration-tasks-submitted] '
u'[transcripts_count=2] [course={}] '
u'[revision=rev-opt-published-only] [video={}]'.format(course_id, self.video_descriptor.location))
)
)
with LogCapture(LOGGER_NAME, level=logging.INFO) as logger:
call_command('migrate_transcripts', '--course-id', six.text_type(self.course.id))
logger.check(
*expected_log
)
def test_migrate_transcripts_exception_logging(self):
"""
Test migrate transcripts exception logging
"""
course_id = six.text_type(self.course_2.id)
expected_log = (
(
'cms.djangoapps.contentstore.tasks', 'INFO',
(u'[Transcript Migration] [run=1] [video-transcripts-migration-process-started-for-course] '
u'[course={}]'.format(course_id))
),
(
'cms.djangoapps.contentstore.tasks', 'INFO',
(u'[Transcript Migration] [run=1] [transcripts-migration-process-started-for-video-transcript] '
u'[revision=rev-opt-published-only] [video={}] [edx_video_id=test_edx_video_id_2] '
u'[language_code=ge]'.format(self.video_descriptor_2.location))
),
(
'cms.djangoapps.contentstore.tasks', 'ERROR',
(u'[Transcript Migration] [run=1] [video-transcript-migration-failed-with-known-exc] '
u'[revision=rev-opt-published-only] [video={}] [edx_video_id=test_edx_video_id_2] '
u'[language_code=ge]'.format(self.video_descriptor_2.location))
),
(
'cms.djangoapps.contentstore.tasks', 'INFO',
(u'[Transcript Migration] [run=1] [transcripts-migration-tasks-submitted] '
u'[transcripts_count=1] [course={}] '
u'[revision=rev-opt-published-only] [video={}]'.format(course_id, self.video_descriptor_2.location))
)
)
with LogCapture(LOGGER_NAME, level=logging.INFO) as logger:
call_command('migrate_transcripts', '--course-id', six.text_type(self.course_2.id), '--commit')
logger.check(
*expected_log
)
@ddt.data(*itertools.product([1, 2], [True, False], [True, False]))
@ddt.unpack
@patch('contentstore.management.commands.migrate_transcripts.log')
def test_migrate_transcripts_batch_size(self, batch_size, commit, all_courses, mock_logger):
"""
Test that migrations across course batches, is working as expected.
"""
migration_settings = TranscriptMigrationSetting.objects.create(
batch_size=batch_size, commit=commit, all_courses=all_courses
)
# Assert the number of job runs and migration enqueued courses.
self.assertEqual(migration_settings.command_run, 0)
self.assertEqual(MigrationEnqueuedCourse.objects.count(), 0)
call_command('migrate_transcripts', '--from-settings')
migration_settings = TranscriptMigrationSetting.current()
# Command run is only incremented if commit=True.
expected_command_run = 1 if commit else 0
self.assertEqual(migration_settings.command_run, expected_command_run)
if all_courses:
mock_logger.info.assert_called_with(
(u'[Transcript Migration] Courses(total): %s, Courses(migrated): %s, '
u'Courses(non-migrated): %s, Courses(migration-in-process): %s'),
2, 0, 2, batch_size
)
# enqueued courses are only persisted if commit=True and job is running for all courses.
enqueued_courses = batch_size if commit and all_courses else 0
self.assertEqual(MigrationEnqueuedCourse.objects.count(), enqueued_courses)

137
cms/djangoapps/contentstore/management/commands/tests/test_video_thumbnails.py
View File

@@ -1,137 +0,0 @@
# -*- coding: utf-8 -*-
"""
Tests for course video thumbnails management command.
"""
import logging
from django.core.management import CommandError, call_command
from django.test import TestCase
from mock import patch
from six import text_type
from testfixtures import LogCapture
from openedx.core.djangoapps.video_config.models import VideoThumbnailSetting
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
LOGGER_NAME = "contentstore.management.commands.video_thumbnails"
def setup_video_thumbnails_config(batch_size=10, commit=False, all_course_videos=False, course_ids=''):
VideoThumbnailSetting.objects.create(
batch_size=batch_size,
commit=commit,
course_ids=course_ids,
all_course_videos=all_course_videos,
videos_per_task=2
)
class TestArgParsing(TestCase):
"""
Tests for parsing arguments for the `video_thumbnails` management command
"""
def test_invalid_course(self):
errstring = "Invalid key specified: <class 'opaque_keys.edx.locator.CourseLocator'>: invalid-course"
setup_video_thumbnails_config(course_ids='invalid-course')
with self.assertRaisesRegex(CommandError, errstring):
call_command('video_thumbnails')
class TestVideoThumbnails(ModuleStoreTestCase):
"""
Tests adding thumbnails to course videos from YouTube
"""
def setUp(self):
""" Common setup """
super(TestVideoThumbnails, self).setUp()
self.course = CourseFactory.create()
self.course_2 = CourseFactory.create()
@patch('edxval.api.get_course_video_ids_with_youtube_profile')
@patch('contentstore.management.commands.video_thumbnails.enqueue_update_thumbnail_tasks')
def test_video_thumbnails_without_commit(self, mock_enqueue_thumbnails, mock_course_videos):
"""
Test that when command is run without commit, correct information is logged.
"""
course_videos = [
(self.course.id, 'super-soaker', 'https://www.youtube.com/watch?v=OscRe3pSP80'),
(self.course_2.id, 'medium-soaker', 'https://www.youtube.com/watch?v=OscRe3pSP81')
]
mock_course_videos.return_value = course_videos
setup_video_thumbnails_config(all_course_videos=True)
with LogCapture(LOGGER_NAME, level=logging.INFO) as logger:
call_command('video_thumbnails')
# Verify that list of course video ids is logged.
logger.check(
(
LOGGER_NAME, 'INFO',
'[Video Thumbnails] Videos(updated): 0, Videos(update-in-process): 2'
),
(
LOGGER_NAME, 'INFO',
u'[video thumbnails] selected course videos: {course_videos} '.format(
course_videos=text_type(course_videos)
)
)
)
# Verify that `enqueue_update_thumbnail_tasks` is not called.
self.assertFalse(mock_enqueue_thumbnails.called)
@patch('edxval.api.get_course_video_ids_with_youtube_profile')
@patch('contentstore.management.commands.video_thumbnails.enqueue_update_thumbnail_tasks')
def test_video_thumbnails_with_commit(self, mock_enqueue_thumbnails, mock_course_videos):
"""
Test that when command is run with with commit, it works as expected.
"""
course_videos = [
(self.course.id, 'super-soaker', 'https://www.youtube.com/watch?v=OscRe3pSP80'),
(self.course_2.id, 'medium-soaker', 'https://www.youtube.com/watch?v=OscRe3pSP81')
]
mock_course_videos.return_value = course_videos
setup_video_thumbnails_config(commit=True, all_course_videos=True)
with LogCapture(LOGGER_NAME, level=logging.INFO) as logger:
call_command('video_thumbnails')
# Verify that command information correctly logged.
logger.check((
LOGGER_NAME, 'INFO',
'[Video Thumbnails] Videos(updated): 0, Videos(update-in-process): 2'
))
# Verify that `enqueue_update_thumbnail_tasks` is called.
self.assertTrue(mock_enqueue_thumbnails.called)
@patch('edxval.api.get_course_video_ids_with_youtube_profile')
@patch('contentstore.video_utils.download_youtube_video_thumbnail') # Mock(side_effect=Exception())
def test_video_thumbnails_scraping_failed(self, mock_scrape_thumbnails, mock_course_videos):
"""
Test that when scraping fails, it is handled correclty.
"""
course_videos = [
(self.course.id, 'super-soaker', 'OscRe3pSP80'),
(self.course_2.id, 'medium-soaker', 'OscRe3pSP81')
]
mock_scrape_thumbnails.side_effect = Exception('error')
mock_course_videos.return_value = course_videos
setup_video_thumbnails_config(commit=True, all_course_videos=True)
tasks_logger = "cms.djangoapps.contentstore.tasks"
with LogCapture(tasks_logger, level=logging.INFO) as logger:
call_command('video_thumbnails')
# Verify that tasks information is correctly logged.
logger.check(
(
tasks_logger, 'ERROR',
(u"[video thumbnails] [run=1] [video-thumbnails-scraping-failed-with-unknown-exc] "
u"[edx_video_id=super-soaker] [youtube_id=OscRe3pSP80] [course={}]".format(self.course.id))
),
(
tasks_logger, 'ERROR',
(u"[video thumbnails] [run=1] [video-thumbnails-scraping-failed-with-unknown-exc] "
u"[edx_video_id=medium-soaker] [youtube_id=OscRe3pSP81] [course={}]".format(self.course_2.id))
)
)

86
cms/djangoapps/contentstore/management/commands/video_thumbnails.py
View File

@@ -1,86 +0,0 @@
"""
Command to scrape thumbnails and add them to the course-videos.
"""
import logging
import edxval.api as edxval_api
from django.core.management import BaseCommand
from django.core.management.base import CommandError
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
from six import text_type
from cms.djangoapps.contentstore.tasks import enqueue_update_thumbnail_tasks
from openedx.core.djangoapps.video_config.models import VideoThumbnailSetting
log = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Example usage:
$ ./manage.py cms video_thumbnails
"""
help = 'Adds thumbnails from YouTube to videos'
def _get_command_options(self):
"""
Returns the command arguments configured via django admin.
"""
command_settings = self._latest_settings()
commit = command_settings.commit
if command_settings.all_course_videos:
course_videos = edxval_api.get_course_video_ids_with_youtube_profile(
offset=command_settings.offset, limit=command_settings.batch_size
)
log.info(
u'[Video Thumbnails] Videos(updated): %s, Videos(update-in-process): %s',
command_settings.offset, len(course_videos),
)
else:
validated_course_ids = self._validate_course_ids(command_settings.course_ids.split())
course_videos = edxval_api.get_course_video_ids_with_youtube_profile(validated_course_ids)
return course_videos, commit
def _validate_course_ids(self, course_ids):
"""
Validate a list of course key strings.
"""
try:
for course_id in course_ids:
CourseKey.from_string(course_id)
return course_ids
except InvalidKeyError as error:
raise CommandError(u'Invalid key specified: {}'.format(text_type(error)))
def _latest_settings(self):
"""
Return the latest version of the VideoThumbnailSetting
"""
return VideoThumbnailSetting.current()
def handle(self, *args, **options):
"""
Invokes the video thumbnail enqueue function.
"""
video_thumbnail_settings = self._latest_settings()
videos_per_task = video_thumbnail_settings.videos_per_task
course_videos, commit = self._get_command_options()
if commit:
command_run = video_thumbnail_settings.increment_run()
enqueue_update_thumbnail_tasks(
course_videos=course_videos,
videos_per_task=videos_per_task,
run=command_run
)
if video_thumbnail_settings.all_course_videos:
video_thumbnail_settings.update_offset()
else:
log.info(u'[video thumbnails] selected course videos: {course_videos} '.format(
course_videos=text_type(course_videos)
))

342
cms/djangoapps/contentstore/tasks.py
View File

@@ -15,7 +15,6 @@ from tempfile import NamedTemporaryFile, mkdtemp
from celery import group
from celery.task import task
from celery.utils.log import get_task_logger
from celery_utils.chordable_django_backend import chord, chord_task
from celery_utils.persist_on_failure import LoggedPersistOnFailureTask
from django.conf import settings
from django.contrib.auth import get_user_model
@@ -76,347 +75,6 @@ User = get_user_model()
LOGGER = get_task_logger(__name__)
FILE_READ_CHUNK = 1024 # bytes
FULL_COURSE_REINDEX_THRESHOLD = 1
DEFAULT_ALL_COURSES = False
DEFAULT_FORCE_UPDATE = False
DEFAULT_COMMIT = False
MIGRATION_LOGS_PREFIX = 'Transcript Migration'
RETRY_DELAY_SECONDS = 30
COURSE_LEVEL_TIMEOUT_SECONDS = 1200
VIDEO_LEVEL_TIMEOUT_SECONDS = 300
def enqueue_update_thumbnail_tasks(course_videos, videos_per_task, run):
"""
Enqueue tasks to update video thumbnails from youtube.
Arguments:
course_videos: A list of tuples, each containing course ID, video ID and youtube ID.
videos_per_task: Number of course videos that can be processed by a single celery task.
run: This tracks the YT thumbnail scraping job runs.
"""
tasks = []
batch_size = len(course_videos)
# Further slice the course-videos batch into chunks on the
# basis of number of course-videos per task.
start = 0
end = videos_per_task
chunks_count = int(ceil(batch_size / float(videos_per_task)))
for __ in range(0, chunks_count): # pylint: disable=C7620
course_videos_chunk = course_videos[start:end]
tasks.append(task_scrape_youtube_thumbnail.s(
course_videos_chunk, run
))
start = end
end += videos_per_task
# Kick off a chord of scraping tasks
callback = task_scrape_youtube_thumbnail_callback.s(
run=run,
batch_size=batch_size,
videos_per_task=videos_per_task,
)
chord(tasks)(callback)
@chord_task(bind=True, routing_key=settings.VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE)
def task_scrape_youtube_thumbnail_callback(self, results, run, # pylint: disable=unused-argument
batch_size, videos_per_task):
"""
Callback for collating the results of yt thumbnails scraping tasks chord.
"""
yt_thumbnails_scraping_tasks_count = len(list(results()))
LOGGER.info(
(u"[video thumbnails] [run=%s] [video-thumbnails-scraping-complete-for-a-batch] [tasks_count=%s] "
u"[batch_size=%s] [videos_per_task=%s]"),
run, yt_thumbnails_scraping_tasks_count, batch_size, videos_per_task
)
@chord_task(
bind=True,
base=LoggedPersistOnFailureTask,
default_retry_delay=RETRY_DELAY_SECONDS,
max_retries=1,
time_limit=COURSE_LEVEL_TIMEOUT_SECONDS,
routing_key=settings.SCRAPE_YOUTUBE_THUMBNAILS_JOB_QUEUE
)
def task_scrape_youtube_thumbnail(self, course_videos, run): # pylint: disable=unused-argument
"""
Task to scrape youtube thumbnails and update them in edxval for the given course-videos.
Arguments:
course_videos: A list of tuples, each containing course ID, video ID and youtube ID.
run: This tracks the YT thumbnail scraping job runs.
"""
for course_id, edx_video_id, youtube_id in course_videos:
try:
scrape_youtube_thumbnail(course_id, edx_video_id, youtube_id)
except Exception: # pylint: disable=broad-except
LOGGER.exception(
(u"[video thumbnails] [run=%s] [video-thumbnails-scraping-failed-with-unknown-exc] "
u"[edx_video_id=%s] [youtube_id=%s] [course=%s]"),
run,
edx_video_id,
youtube_id,
course_id
)
continue
@chord_task(bind=True, routing_key=settings.VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE)
def task_status_callback(self, results, revision, # pylint: disable=unused-argument
course_id, command_run, video_location):
"""
Callback for collating the results of chord.
"""
transcript_tasks_count = len(list(results()))
LOGGER.info(
(u"[%s] [run=%s] [video-transcripts-migration-complete-for-a-video] [tasks_count=%s] [course_id=%s] "
u"[revision=%s] [video=%s]"),
MIGRATION_LOGS_PREFIX, command_run, transcript_tasks_count, course_id, revision, video_location
)
def enqueue_async_migrate_transcripts_tasks(course_keys,
command_run,
force_update=DEFAULT_FORCE_UPDATE,
commit=DEFAULT_COMMIT):
"""
Fires new Celery tasks for all the input courses or for all courses.
Arguments:
course_keys: Command line course ids as list of CourseKey objects
command_run: A positive number indicating the run counts for transcripts migrations
force_update: Overwrite file in S3. Default is False
commit: Update S3 or dry-run the command to see which transcripts will be affected. Default is False
"""
kwargs = {
'force_update': force_update,
'commit': commit,
'command_run': command_run
}
group([
async_migrate_transcript.s(text_type(course_key), **kwargs)
for course_key in course_keys
])()
def get_course_videos(course_key):
"""
Returns all videos in a course as list.
Arguments:
course_key: CourseKey object
"""
all_videos = {}
store = modulestore()
# include published videos of the course.
all_videos[ModuleStoreEnum.RevisionOption.published_only] = store.get_items(
course_key,
qualifiers={'category': 'video'},
revision=ModuleStoreEnum.RevisionOption.published_only,
include_orphans=False
)
# include draft videos of the course.
all_videos[ModuleStoreEnum.RevisionOption.draft_only] = store.get_items(
course_key,
qualifiers={'category': 'video'},
revision=ModuleStoreEnum.RevisionOption.draft_only,
include_orphans=False
)
return all_videos
@chord_task(
bind=True,
base=LoggedPersistOnFailureTask,
default_retry_delay=RETRY_DELAY_SECONDS,
max_retries=1,
time_limit=COURSE_LEVEL_TIMEOUT_SECONDS,
routing_key=settings.VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE
)
def async_migrate_transcript(self, course_key, **kwargs): # pylint: disable=unused-argument
"""
Migrates the transcripts of all videos in a course as a new celery task.
"""
force_update = kwargs['force_update']
command_run = kwargs['command_run']
course_videos = get_course_videos(CourseKey.from_string(course_key))
LOGGER.info(
u"[%s] [run=%s] [video-transcripts-migration-process-started-for-course] [course=%s]",
MIGRATION_LOGS_PREFIX, command_run, course_key
)
for revision, videos in course_videos.items():
for video in videos:
# Gather transcripts from a video block.
all_transcripts = {}
if video.transcripts is not None:
all_transcripts.update(video.transcripts)
english_transcript = video.sub
if english_transcript:
all_transcripts.update({'en': video.sub})
sub_tasks = []
video_location = text_type(video.location)
for lang in all_transcripts:
sub_tasks.append(async_migrate_transcript_subtask.s(
video_location, revision, lang, force_update, **kwargs
))
if sub_tasks:
callback = task_status_callback.s(
revision=revision,
course_id=course_key,
command_run=command_run,
video_location=video_location
)
chord(sub_tasks)(callback)
LOGGER.info(
(u"[%s] [run=%s] [transcripts-migration-tasks-submitted] "
u"[transcripts_count=%s] [course=%s] [revision=%s] [video=%s]"),
MIGRATION_LOGS_PREFIX, command_run, len(sub_tasks), course_key, revision, video_location
)
else:
LOGGER.info(
u"[%s] [run=%s] [no-video-transcripts] [course=%s] [revision=%s] [video=%s]",
MIGRATION_LOGS_PREFIX, command_run, course_key, revision, video_location
)
def save_transcript_to_storage(command_run, edx_video_id, language_code, transcript_content, file_format, force_update):
"""
Pushes a given transcript's data to django storage.
Arguments:
command_run: A positive integer indicating the current run
edx_video_id: video ID
language_code: language code
transcript_content: content of the transcript
file_format: format of the transcript file
force_update: tells whether it needs to perform force update in
case of an existing transcript for the given video.
"""
transcript_present = is_transcript_available(video_id=edx_video_id, language_code=language_code)
if transcript_present and force_update:
create_or_update_video_transcript(
edx_video_id,
language_code,
dict({'file_format': file_format}),
ContentFile(transcript_content)
)
elif not transcript_present:
create_video_transcript(
edx_video_id,
language_code,
file_format,
ContentFile(transcript_content)
)
else:
LOGGER.info(
u"[%s] [run=%s] [do-not-override-existing-transcript] [edx_video_id=%s] [language_code=%s]",
MIGRATION_LOGS_PREFIX, command_run, edx_video_id, language_code
)
@chord_task(
bind=True,
base=LoggedPersistOnFailureTask,
default_retry_delay=RETRY_DELAY_SECONDS,
max_retries=2,
time_limit=VIDEO_LEVEL_TIMEOUT_SECONDS,
routing_key=settings.VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE
)
def async_migrate_transcript_subtask(self, *args, **kwargs): # pylint: disable=unused-argument
"""
Migrates a transcript of a given video in a course as a new celery task.
"""
success, failure = 'Success', 'Failure'
video_location, revision, language_code, force_update = args
command_run = kwargs['command_run']
store = modulestore()
video = store.get_item(usage_key=BlockUsageLocator.from_string(video_location), revision=revision)
edx_video_id = clean_video_id(video.edx_video_id)
if not kwargs['commit']:
LOGGER.info(
(u'[%s] [run=%s] [video-transcript-will-be-migrated] '
u'[revision=%s] [video=%s] [edx_video_id=%s] [language_code=%s]'),
MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
)
return success
LOGGER.info(
(u'[%s] [run=%s] [transcripts-migration-process-started-for-video-transcript] [revision=%s] '
u'[video=%s] [edx_video_id=%s] [language_code=%s]'),
MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
)
try:
transcripts_info = video.get_transcripts_info()
transcript_content, _, _ = get_transcript_from_contentstore(
video=video,
language=language_code,
output_format=Transcript.SJSON,
transcripts_info=transcripts_info,
)
is_video_valid = edx_video_id and is_video_available(edx_video_id)
if not is_video_valid:
edx_video_id = create_external_video('external-video')
video.edx_video_id = edx_video_id
# determine branch published/draft
branch_setting = (
ModuleStoreEnum.Branch.published_only
if revision == ModuleStoreEnum.RevisionOption.published_only else
ModuleStoreEnum.Branch.draft_preferred
)
with store.branch_setting(branch_setting):
store.update_item(video, ModuleStoreEnum.UserID.mgmt_command)
LOGGER.info(
u'[%s] [run=%s] [generated-edx-video-id] [revision=%s] [video=%s] [edx_video_id=%s] [language_code=%s]',
MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
)
save_transcript_to_storage(
command_run=command_run,
edx_video_id=edx_video_id,
language_code=language_code,
transcript_content=transcript_content,
file_format=Transcript.SJSON,
force_update=force_update,
)
except (NotFoundError, TranscriptsGenerationException, ValCannotCreateError):
LOGGER.exception(
(u'[%s] [run=%s] [video-transcript-migration-failed-with-known-exc] [revision=%s] [video=%s] '
u'[edx_video_id=%s] [language_code=%s]'),
MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
)
return failure
except Exception:
LOGGER.exception(
(u'[%s] [run=%s] [video-transcript-migration-failed-with-unknown-exc] [revision=%s] '
u'[video=%s] [edx_video_id=%s] [language_code=%s]'),
MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
)
raise
LOGGER.info(
(u'[%s] [run=%s] [video-transcript-migration-succeeded-for-a-video] [revision=%s] '
u'[video=%s] [edx_video_id=%s] [language_code=%s]'),
MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
)
return success
def clone_instance(instance, field_values):

12
requirements/edx/base.txt
View File

@@ -55,7 +55,7 @@ defusedxml==0.5.0 # via -r requirements/edx/base.in, djangorestframework
git+https://github.com/django-compressor/django-appconf@1526a842ee084b791aa66c931b3822091a442853#egg=django-appconf # via -r requirements/edx/github.in, django-statici18n
django-babel-underscore==0.5.2 # via -r requirements/edx/base.in
django-babel==0.6.2 # via django-babel-underscore
git+https://github.com/edx/django-celery.git@756cb57aad765cb2b0d37372c1855b8f5f37e6b0#egg=django-celery==3.2.1+edx.2 # via -r requirements/edx/github.in, edx-celeryutils, super-csv
git+https://github.com/edx/django-celery.git@756cb57aad765cb2b0d37372c1855b8f5f37e6b0#egg=django-celery==3.2.1+edx.2 # via -r requirements/edx/github.in
django-classy-tags==1.0.0 # via django-sekizai
django-config-models==2.0.0 # via -r requirements/edx/base.in, edx-enterprise
django-cors-headers==2.5.3 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.in
@@ -97,13 +97,13 @@ edx-analytics-data-api-client==0.15.3 # via -r requirements/edx/base.in
edx-api-doc-tools==1.0.2 # via -r requirements/edx/base.in
edx-bulk-grades==0.6.6 # via -r requirements/edx/base.in, staff-graded-xblock
edx-ccx-keys==1.0.0 # via -r requirements/edx/base.in
edx-celeryutils==0.3.2 # via -r requirements/edx/base.in, super-csv
edx-celeryutils==0.4.0 # via -r requirements/edx/base.in, super-csv
edx-completion==3.1.1 # via -r requirements/edx/base.in
edx-django-release-util==0.3.6 # via -r requirements/edx/base.in
edx-django-sites-extensions==2.4.3 # via -r requirements/edx/base.in
edx-django-utils==3.0 # via -r requirements/edx/base.in, django-config-models, edx-drf-extensions, edx-enterprise, edx-rest-api-client
edx-drf-extensions==3.0.0 # via -r requirements/edx/base.in, edx-completion, edx-enterprise, edx-organizations, edx-proctoring, edx-rbac, edx-when, edxval
edx-enterprise==2.5.0 # via -r requirements/edx/base.in
edx-enterprise==2.5.1 # via -r requirements/edx/base.in
edx-i18n-tools==0.5.0 # via ora2
edx-milestones==0.2.6 # via -r requirements/edx/base.in
edx-opaque-keys[django]==2.0.1 # via -r requirements/edx/paver.txt, edx-bulk-grades, edx-ccx-keys, edx-completion, edx-drf-extensions, edx-enterprise, edx-milestones, edx-organizations, edx-proctoring, edx-user-state-client, edx-when, xmodule
@@ -168,7 +168,7 @@ numpy==1.18.1 # via calc, chem, scipy
git+https://github.com/joestump/python-oauth2.git@b94f69b1ad195513547924e380d9265133e995fa#egg=oauth2 # via -r requirements/edx/github.in
oauthlib==2.1.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.in, django-oauth-toolkit, lti-consumer-xblock, requests-oauthlib, social-auth-core
git+https://github.com/edx/edx-ora2.git@2.6.17#egg=ora2==2.6.17 # via -r requirements/edx/github.in
packaging==20.1 # via drf-yasg
packaging==20.3 # via drf-yasg
path.py==12.4.0 # via edx-enterprise, edx-i18n-tools, ora2, xmodule
path==13.1.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/paver.txt, path.py
pathtools==0.1.2 # via -r requirements/edx/paver.txt, watchdog
@@ -218,7 +218,7 @@ sailthru-client==2.2.3 # via -r requirements/edx/base.in, edx-ace
scipy==1.4.1 # via calc, chem
semantic-version==2.8.4 # via edx-drf-extensions
shapely==1.7.0 # via -r requirements/edx/base.in
shortuuid==0.5.0 # via -r requirements/edx/base.in
shortuuid==0.5.1 # via -r requirements/edx/base.in
simplejson==3.17.0 # via -r requirements/edx/base.in, sailthru-client, super-csv, xblock-utils
six==1.14.0 # via -r requirements/edx/../edx-sandbox/shared.txt, -r requirements/edx/base.in, -r requirements/edx/paver.txt, analytics-python, bleach, calc, cryptography, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, event-tracking, fs, fs-s3fs, help-tokens, html5lib, isodate, libsass, mock, nltk, packaging, paver, pycontracts, pyjwkest, python-dateutil, python-memcached, python-swiftclient, social-auth-app-django, social-auth-core, stevedore, xblock
slumber==0.7.1 # via edx-bulk-grades, edx-enterprise, edx-rest-api-client
@@ -229,7 +229,7 @@ soupsieve==2.0 # via beautifulsoup4
sqlparse==0.3.1 # via -r requirements/edx/base.in
staff-graded-xblock==0.7 # via -r requirements/edx/base.in
stevedore==1.32.0 # via -r requirements/edx/base.in, -r requirements/edx/paver.txt, code-annotations, edx-ace, edx-enterprise, edx-opaque-keys
super-csv==0.9.6 # via -r requirements/edx/base.in, edx-bulk-grades
super-csv==0.9.7 # via -r requirements/edx/base.in, edx-bulk-grades
sympy==1.5.1 # via symmath
testfixtures==6.14.0 # via edx-enterprise
text-unidecode==1.3 # via python-slugify

14
requirements/edx/development.txt
View File

@@ -66,7 +66,7 @@ distlib==0.3.0 # via -r requirements/edx/testing.txt, virtualenv
git+https://github.com/django-compressor/django-appconf@1526a842ee084b791aa66c931b3822091a442853#egg=django-appconf # via -r requirements/edx/testing.txt, django-statici18n
django-babel-underscore==0.5.2 # via -r requirements/edx/testing.txt
django-babel==0.6.2 # via -r requirements/edx/testing.txt, django-babel-underscore
git+https://github.com/edx/django-celery.git@756cb57aad765cb2b0d37372c1855b8f5f37e6b0#egg=django-celery==3.2.1+edx.2 # via -r requirements/edx/testing.txt, edx-celeryutils, super-csv
git+https://github.com/edx/django-celery.git@756cb57aad765cb2b0d37372c1855b8f5f37e6b0#egg=django-celery==3.2.1+edx.2 # via -r requirements/edx/testing.txt
django-classy-tags==1.0.0 # via -r requirements/edx/testing.txt, django-sekizai
django-config-models==2.0.0 # via -r requirements/edx/testing.txt, edx-enterprise
django-cors-headers==2.5.3 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt
@@ -109,13 +109,13 @@ edx-analytics-data-api-client==0.15.3 # via -r requirements/edx/testing.txt
edx-api-doc-tools==1.0.2 # via -r requirements/edx/testing.txt
edx-bulk-grades==0.6.6 # via -r requirements/edx/testing.txt, staff-graded-xblock
edx-ccx-keys==1.0.0 # via -r requirements/edx/testing.txt
edx-celeryutils==0.3.2 # via -r requirements/edx/testing.txt, super-csv
edx-celeryutils==0.4.0 # via -r requirements/edx/testing.txt, super-csv
edx-completion==3.1.1 # via -r requirements/edx/testing.txt
edx-django-release-util==0.3.6 # via -r requirements/edx/testing.txt
edx-django-sites-extensions==2.4.3 # via -r requirements/edx/testing.txt
edx-django-utils==3.0 # via -r requirements/edx/testing.txt, django-config-models, edx-drf-extensions, edx-enterprise, edx-rest-api-client
edx-drf-extensions==3.0.0 # via -r requirements/edx/testing.txt, edx-completion, edx-enterprise, edx-organizations, edx-proctoring, edx-rbac, edx-when, edxval
edx-enterprise==2.5.0 # via -r requirements/edx/testing.txt
edx-enterprise==2.5.1 # via -r requirements/edx/testing.txt
edx-i18n-tools==0.5.0 # via -r requirements/edx/testing.txt, ora2
edx-lint==1.4.1 # via -r requirements/edx/testing.txt
edx-milestones==0.2.6 # via -r requirements/edx/testing.txt
@@ -202,7 +202,7 @@ numpy==1.18.1 # via -r requirements/edx/testing.txt, calc, chem, pan
git+https://github.com/joestump/python-oauth2.git@b94f69b1ad195513547924e380d9265133e995fa#egg=oauth2 # via -r requirements/edx/testing.txt
oauthlib==2.1.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, django-oauth-toolkit, lti-consumer-xblock, requests-oauthlib, social-auth-core
git+https://github.com/edx/edx-ora2.git@2.6.17#egg=ora2==2.6.17 # via -r requirements/edx/testing.txt
packaging==20.1 # via -r requirements/edx/testing.txt, drf-yasg, pytest, sphinx, tox
packaging==20.3 # via -r requirements/edx/testing.txt, drf-yasg, pytest, sphinx, tox
pandas==0.22.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt
path.py==12.4.0 # via -r requirements/edx/testing.txt, edx-enterprise, edx-i18n-tools, ora2, xmodule
path==13.1.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, path.py
@@ -277,7 +277,7 @@ scipy==1.4.1 # via -r requirements/edx/testing.txt, calc, chem
selenium==3.141.0 # via -r requirements/edx/testing.txt, bok-choy
semantic-version==2.8.4 # via -r requirements/edx/testing.txt, edx-drf-extensions
shapely==1.7.0 # via -r requirements/edx/testing.txt
shortuuid==0.5.0 # via -r requirements/edx/testing.txt
shortuuid==0.5.1 # via -r requirements/edx/testing.txt
simplejson==3.17.0 # via -r requirements/edx/testing.txt, sailthru-client, super-csv, xblock-utils
singledispatch==3.4.0.3 # via -r requirements/edx/testing.txt
six==1.14.0 # via -r requirements/edx/pip-tools.txt, -r requirements/edx/testing.txt, analytics-python, astroid, bleach, bok-choy, calc, cryptography, diff-cover, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-lint, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, edx-sphinx-theme, event-tracking, freezegun, fs, fs-s3fs, help-tokens, html5lib, httpretty, isodate, jsonschema, libsass, mando, mock, nltk, packaging, pathlib2, paver, pip-tools, pycontracts, pyjwkest, pytest-xdist, python-dateutil, python-memcached, python-swiftclient, singledispatch, social-auth-app-django, social-auth-core, sphinxcontrib-httpdomain, stevedore, tox, transifex-client, virtualenv, xblock
@@ -287,7 +287,7 @@ social-auth-core==3.2.0 # via -r requirements/edx/testing.txt, social-auth-app
git+https://github.com/jazzband/sorl-thumbnail.git@13bedfb7d2970809eda597e3ef79318a6fa80ac2#egg=sorl-thumbnail # via -r requirements/edx/testing.txt
sortedcontainers==2.1.0 # via -r requirements/edx/testing.txt, pdfminer.six
soupsieve==2.0 # via -r requirements/edx/testing.txt, beautifulsoup4
sphinx==2.4.3 # via edx-sphinx-theme, sphinxcontrib-httpdomain
sphinx==2.4.4 # via edx-sphinx-theme, sphinxcontrib-httpdomain
sphinxcontrib-applehelp==1.0.2 # via sphinx
sphinxcontrib-devhelp==1.0.2 # via sphinx
sphinxcontrib-htmlhelp==1.0.3 # via sphinx
@@ -299,7 +299,7 @@ sphinxcontrib-serializinghtml==1.1.4 # via sphinx
sqlparse==0.3.1 # via -r requirements/edx/testing.txt, django-debug-toolbar
staff-graded-xblock==0.7 # via -r requirements/edx/testing.txt
stevedore==1.32.0 # via -r requirements/edx/testing.txt, code-annotations, edx-ace, edx-enterprise, edx-opaque-keys
super-csv==0.9.6 # via -r requirements/edx/testing.txt, edx-bulk-grades
super-csv==0.9.7 # via -r requirements/edx/testing.txt, edx-bulk-grades
sympy==1.5.1 # via -r requirements/edx/testing.txt, symmath
testfixtures==6.14.0 # via -r requirements/edx/testing.txt, edx-enterprise
text-unidecode==1.3 # via -r requirements/edx/testing.txt, faker, python-slugify

12
requirements/edx/testing.txt
View File

@@ -65,7 +65,7 @@ distlib==0.3.0 # via virtualenv
git+https://github.com/django-compressor/django-appconf@1526a842ee084b791aa66c931b3822091a442853#egg=django-appconf # via -r requirements/edx/base.txt, django-statici18n
django-babel-underscore==0.5.2 # via -r requirements/edx/base.txt
django-babel==0.6.2 # via -r requirements/edx/base.txt, django-babel-underscore
git+https://github.com/edx/django-celery.git@756cb57aad765cb2b0d37372c1855b8f5f37e6b0#egg=django-celery==3.2.1+edx.2 # via -r requirements/edx/base.txt, edx-celeryutils, super-csv
git+https://github.com/edx/django-celery.git@756cb57aad765cb2b0d37372c1855b8f5f37e6b0#egg=django-celery==3.2.1+edx.2 # via -r requirements/edx/base.txt
django-classy-tags==1.0.0 # via -r requirements/edx/base.txt, django-sekizai
django-config-models==2.0.0 # via -r requirements/edx/base.txt, edx-enterprise
django-cors-headers==2.5.3 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt
@@ -105,13 +105,13 @@ edx-analytics-data-api-client==0.15.3 # via -r requirements/edx/base.txt
edx-api-doc-tools==1.0.2 # via -r requirements/edx/base.txt
edx-bulk-grades==0.6.6 # via -r requirements/edx/base.txt, staff-graded-xblock
edx-ccx-keys==1.0.0 # via -r requirements/edx/base.txt
edx-celeryutils==0.3.2 # via -r requirements/edx/base.txt, super-csv
edx-celeryutils==0.4.0 # via -r requirements/edx/base.txt, super-csv
edx-completion==3.1.1 # via -r requirements/edx/base.txt
edx-django-release-util==0.3.6 # via -r requirements/edx/base.txt
edx-django-sites-extensions==2.4.3 # via -r requirements/edx/base.txt
edx-django-utils==3.0 # via -r requirements/edx/base.txt, django-config-models, edx-drf-extensions, edx-enterprise, edx-rest-api-client
edx-drf-extensions==3.0.0 # via -r requirements/edx/base.txt, edx-completion, edx-enterprise, edx-organizations, edx-proctoring, edx-rbac, edx-when, edxval
edx-enterprise==2.5.0 # via -r requirements/edx/base.txt
edx-enterprise==2.5.1 # via -r requirements/edx/base.txt
edx-i18n-tools==0.5.0 # via -r requirements/edx/base.txt, -r requirements/edx/testing.in, ora2
edx-lint==1.4.1 # via -r requirements/edx/testing.in
edx-milestones==0.2.6 # via -r requirements/edx/base.txt
@@ -193,7 +193,7 @@ numpy==1.18.1 # via -r requirements/edx/base.txt, -r requirements/ed
git+https://github.com/joestump/python-oauth2.git@b94f69b1ad195513547924e380d9265133e995fa#egg=oauth2 # via -r requirements/edx/base.txt
oauthlib==2.1.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, django-oauth-toolkit, lti-consumer-xblock, requests-oauthlib, social-auth-core
git+https://github.com/edx/edx-ora2.git@2.6.17#egg=ora2==2.6.17 # via -r requirements/edx/base.txt
packaging==20.1 # via -r requirements/edx/base.txt, drf-yasg, pytest, tox
packaging==20.3 # via -r requirements/edx/base.txt, drf-yasg, pytest, tox
pandas==0.22.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/coverage.txt
path.py==12.4.0 # via -r requirements/edx/base.txt, edx-enterprise, edx-i18n-tools, ora2, xmodule
path==13.1.0 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, path.py
@@ -265,7 +265,7 @@ scipy==1.4.1 # via -r requirements/edx/base.txt, calc, chem
selenium==3.141.0 # via -r requirements/edx/testing.in, bok-choy
semantic-version==2.8.4 # via -r requirements/edx/base.txt, edx-drf-extensions
shapely==1.7.0 # via -r requirements/edx/base.txt
shortuuid==0.5.0 # via -r requirements/edx/base.txt
shortuuid==0.5.1 # via -r requirements/edx/base.txt
simplejson==3.17.0 # via -r requirements/edx/base.txt, sailthru-client, super-csv, xblock-utils
singledispatch==3.4.0.3 # via -r requirements/edx/testing.in
six==1.14.0 # via -r requirements/edx/base.txt, -r requirements/edx/coverage.txt, analytics-python, astroid, bleach, bok-choy, calc, cryptography, diff-cover, django-appconf, django-classy-tags, django-countries, django-pyfs, django-sekizai, django-simple-history, django-statici18n, drf-yasg, edx-ace, edx-ccx-keys, edx-django-release-util, edx-drf-extensions, edx-enterprise, edx-i18n-tools, edx-lint, edx-milestones, edx-opaque-keys, edx-rbac, edx-search, event-tracking, freezegun, fs, fs-s3fs, help-tokens, html5lib, httpretty, isodate, libsass, mando, mock, nltk, packaging, pathlib2, paver, pycontracts, pyjwkest, pytest-xdist, python-dateutil, python-memcached, python-swiftclient, singledispatch, social-auth-app-django, social-auth-core, stevedore, tox, transifex-client, virtualenv, xblock
@@ -277,7 +277,7 @@ soupsieve==2.0 # via -r requirements/edx/base.txt, beautifulsoup4
sqlparse==0.3.1 # via -r requirements/edx/base.txt
staff-graded-xblock==0.7 # via -r requirements/edx/base.txt
stevedore==1.32.0 # via -r requirements/edx/base.txt, code-annotations, edx-ace, edx-enterprise, edx-opaque-keys
super-csv==0.9.6 # via -r requirements/edx/base.txt, edx-bulk-grades
super-csv==0.9.7 # via -r requirements/edx/base.txt, edx-bulk-grades
sympy==1.5.1 # via -r requirements/edx/base.txt, symmath
testfixtures==6.14.0 # via -r requirements/edx/base.txt, -r requirements/edx/testing.in, edx-enterprise
text-unidecode==1.3 # via -r requirements/edx/base.txt, faker, python-slugify