diff --git a/cms/djangoapps/contentstore/admin.py b/cms/djangoapps/contentstore/admin.py new file mode 100644 index 0000000000..eb1b756ff5 --- /dev/null +++ b/cms/djangoapps/contentstore/admin.py @@ -0,0 +1,10 @@ +""" +Admin site bindings for contentstore +""" + +from django.contrib import admin + +from config_models.admin import ConfigurationModelAdmin +from contentstore.models import VideoEncodingDownloadConfig + +admin.site.register(VideoEncodingDownloadConfig, ConfigurationModelAdmin) diff --git a/cms/djangoapps/contentstore/migrations/0001_initial.py b/cms/djangoapps/contentstore/migrations/0001_initial.py new file mode 100644 index 0000000000..0ece625496 --- /dev/null +++ b/cms/djangoapps/contentstore/migrations/0001_initial.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'VideoEncodingDownloadConfig' + db.create_table('contentstore_videoencodingdownloadconfig', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('change_date', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), + ('changed_by', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'], null=True, on_delete=models.PROTECT)), + ('enabled', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('profile_whitelist', self.gf('django.db.models.fields.TextField')(blank=True)), + ('status_whitelist', self.gf('django.db.models.fields.TextField')(blank=True)), + )) + db.send_create_signal('contentstore', ['VideoEncodingDownloadConfig']) + + + def backwards(self, orm): + # Deleting model 'VideoEncodingDownloadConfig' + db.delete_table('contentstore_videoencodingdownloadconfig') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contentstore.videoencodingdownloadconfig': { + 'Meta': {'object_name': 'VideoEncodingDownloadConfig'}, + 'change_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'changed_by': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'on_delete': 'models.PROTECT'}), + 'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'profile_whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'status_whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + } + } + + complete_apps = ['contentstore'] \ No newline at end of file diff --git a/cms/djangoapps/contentstore/migrations/__init__.py b/cms/djangoapps/contentstore/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cms/djangoapps/contentstore/models.py b/cms/djangoapps/contentstore/models.py new file mode 100644 index 0000000000..6cd9b68e37 --- /dev/null +++ b/cms/djangoapps/contentstore/models.py @@ -0,0 +1,32 @@ +""" +Models for contentstore +""" + +from django.db.models.fields import TextField + +from config_models.models import ConfigurationModel + + +class VideoEncodingDownloadConfig(ConfigurationModel): + """Configuration for what to include in video encoding downloads""" + profile_whitelist = TextField( + blank=True, + help_text="A comma-separated list of names of profiles to include in video encoding downloads" + ) + status_whitelist = TextField( + blank=True, + help_text="A comma-separated list of status values; only videos with these status values will be included in video encoding downloads" + ) + + @classmethod + def get_profile_whitelist(cls): + """Get the list of profiles to include in the encoding download""" + return [profile for profile in cls.current().profile_whitelist.split(",") if profile] + + @classmethod + def get_status_whitelist(cls): + """ + Get the list of status values to include files for in the encoding + download + """ + return [status for status in cls.current().status_whitelist.split(",") if status] diff --git a/cms/djangoapps/contentstore/views/tests/test_videos.py b/cms/djangoapps/contentstore/views/tests/test_videos.py index b91db32f88..ee498a6442 100644 --- a/cms/djangoapps/contentstore/views/tests/test_videos.py +++ b/cms/djangoapps/contentstore/views/tests/test_videos.py @@ -1,43 +1,59 @@ +#-*- coding: utf-8 -*- """ Unit tests for video-related REST APIs. """ # pylint: disable=attribute-defined-outside-init +import csv import json import dateutil.parser import re +from StringIO import StringIO from django.conf import settings from django.test.utils import override_settings from mock import Mock, patch -from edxval.api import create_video, get_video_info +from edxval.api import create_profile, create_video, get_video_info +from contentstore.models import VideoEncodingDownloadConfig from contentstore.views.videos import KEY_EXPIRATION_IN_SECONDS, VIDEO_ASSET_TYPE from contentstore.tests.utils import CourseTestCase from contentstore.utils import reverse_course_url from xmodule.assetstore import AssetMetadata from xmodule.modulestore.django import modulestore +from xmodule.modulestore.tests.factories import CourseFactory -@patch.dict("django.conf.settings.FEATURES", {"ENABLE_VIDEO_UPLOAD_PIPELINE": True}) -@override_settings(VIDEO_UPLOAD_PIPELINE={"BUCKET": "test_bucket", "ROOT_PATH": "test_root"}) -class VideoUploadTestCase(CourseTestCase): +class VideoUploadTestMixin(object): """ - Test cases for the video upload page + Test cases for the video upload feature """ - @staticmethod - def get_url_for_course_key(course_key): + def get_url_for_course_key(self, course_key): """Return video handler URL for the given course""" - return reverse_course_url("videos_handler", course_key) + return reverse_course_url(self.VIEW_NAME, course_key) def setUp(self): - super(VideoUploadTestCase, self).setUp() - self.url = VideoUploadTestCase.get_url_for_course_key(self.course.id) + super(VideoUploadTestMixin, self).setUp() + self.url = self.get_url_for_course_key(self.course.id) self.test_token = "test_token" self.course.video_upload_pipeline = { "course_video_upload_token": self.test_token, } self.save_course() + self.profiles = [ + { + "profile_name": "profile1", + "extension": "mp4", + "width": 640, + "height": 480, + }, + { + "profile_name": "profile2", + "extension": "mp4", + "width": 1920, + "height": 1080, + }, + ] self.previous_uploads = [ { "edx_video_id": "test1", @@ -51,9 +67,38 @@ class VideoUploadTestCase(CourseTestCase): "client_video_id": "test2.mp4", "duration": 128.0, "status": "file_complete", - "encoded_videos": [], - } + "encoded_videos": [ + { + "profile": "profile1", + "url": "http://example.com/profile1/test2.mp4", + "file_size": 1600, + "bitrate": 100, + }, + { + "profile": "profile2", + "url": "http://example.com/profile2/test2.mov", + "file_size": 16000, + "bitrate": 1000, + }, + ], + }, + { + "edx_video_id": "non-ascii", + "client_video_id": u"nón-ascii-näme.mp4", + "duration": 256.0, + "status": "file_delivered", + "encoded_videos": [ + { + "profile": "profile1", + "url": u"http://example.com/profile1/nón-ascii-näme.mp4", + "file_size": 3200, + "bitrate": 100, + }, + ] + }, ] + for profile in self.profiles: + create_profile(profile) for video in self.previous_uploads: create_video(video) modulestore().save_asset_metadata( @@ -63,6 +108,14 @@ class VideoUploadTestCase(CourseTestCase): self.user.id ) + def _get_previous_upload(self, edx_video_id): + """Returns the previous upload with the given video id.""" + return next( + video + for video in self.previous_uploads + if video["edx_video_id"] == edx_video_id + ) + def test_anon_user(self): self.client.logout() response = self.client.get(self.url) @@ -74,7 +127,7 @@ class VideoUploadTestCase(CourseTestCase): def test_invalid_course_key(self): response = self.client.get( - VideoUploadTestCase.get_url_for_course_key("Non/Existent/Course") + self.get_url_for_course_key("Non/Existent/Course") ) self.assertEqual(response.status_code, 404) @@ -96,17 +149,21 @@ class VideoUploadTestCase(CourseTestCase): self.save_course() self.assertEqual(self.client.get(self.url).status_code, 404) + +@patch.dict("django.conf.settings.FEATURES", {"ENABLE_VIDEO_UPLOAD_PIPELINE": True}) +@override_settings(VIDEO_UPLOAD_PIPELINE={"BUCKET": "test_bucket", "ROOT_PATH": "test_root"}) +class VideosHandlerTestCase(VideoUploadTestMixin, CourseTestCase): + """Test cases for the main video upload endpoint""" + + VIEW_NAME = "videos_handler" + def test_get_json(self): response = self.client.get_json(self.url) self.assertEqual(response.status_code, 200) response_videos = json.loads(response.content)["videos"] self.assertEqual(len(response_videos), len(self.previous_uploads)) for response_video in response_videos: - original_video = dict( - next( - video for video in self.previous_uploads if video["edx_video_id"] == response_video["edx_video_id"] - ) - ) + original_video = self._get_previous_upload(response_video["edx_video_id"]) self.assertEqual( set(response_video.keys()), set(["edx_video_id", "client_video_id", "created", "duration", "status"]) @@ -191,6 +248,7 @@ class VideoUploadTestCase(CourseTestCase): json.dumps({"files": files}), content_type="application/json" ) + self.assertEqual(response.status_code, 200) response_obj = json.loads(response.content) mock_conn.assert_called_once_with(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) @@ -243,3 +301,87 @@ class VideoUploadTestCase(CourseTestCase): response_file = response_obj["files"][i] self.assertEqual(response_file["file_name"], file_info["file_name"]) self.assertEqual(response_file["upload_url"], mock_key_instance.generate_url()) + + +@patch.dict("django.conf.settings.FEATURES", {"ENABLE_VIDEO_UPLOAD_PIPELINE": True}) +@override_settings(VIDEO_UPLOAD_PIPELINE={"BUCKET": "test_bucket", "ROOT_PATH": "test_root"}) +class VideoUrlsCsvTestCase(VideoUploadTestMixin, CourseTestCase): + """Test cases for the CSV download endpoint for video uploads""" + + VIEW_NAME = "video_encodings_download" + + def setUp(self): + super(VideoUrlsCsvTestCase, self).setUp() + VideoEncodingDownloadConfig( + profile_whitelist="profile1", + status_whitelist="file_delivered,file_complete" + ).save() + + def _check_csv_response(self, expected_video_ids, expected_profiles): + """ + Check that the response is a valid CSV response containing rows + corresponding to expected_video_ids. + """ + response = self.client.get(self.url) + self.assertEqual(response.status_code, 200) + self.assertEqual( + response["Content-Disposition"], + "attachment; filename={course}_video_urls.csv".format(course=self.course.id.course) + ) + response_reader = StringIO(response.content) + reader = csv.DictReader(response_reader, dialect=csv.excel) + self.assertEqual( + reader.fieldnames, + ( + ["Name", "Duration", "Date Added", "Video ID"] + + ["{} URL".format(profile) for profile in expected_profiles] + ) + ) + actual_video_ids = [] + for row in reader: + response_video = { + key.decode("utf-8"): value.decode("utf-8") for key, value in row.items() + } + actual_video_ids.append(response_video["Video ID"]) + original_video = self._get_previous_upload(response_video["Video ID"]) + self.assertEqual(response_video["Name"], original_video["client_video_id"]) + self.assertEqual(response_video["Video ID"], original_video["edx_video_id"]) + for profile in expected_profiles: + response_profile_url = response_video["{} URL".format(profile)] + original_encoded_for_profile = next( + ( + original_encoded + for original_encoded in original_video["encoded_videos"] + if original_encoded["profile"] == profile + ), + None + ) + if original_encoded_for_profile: + self.assertEqual(response_profile_url, original_encoded_for_profile["url"]) + else: + self.assertEqual(response_profile_url, "") + self.assertEqual(set(actual_video_ids), set(expected_video_ids)) + + def test_basic(self): + self._check_csv_response(["test2", "non-ascii"], ["profile1"]) + + def test_config(self): + VideoEncodingDownloadConfig( + profile_whitelist="profile1,profile2", + status_whitelist="file_delivered,file_complete,transcode_active" + ).save() + self._check_csv_response(["test1", "test2", "non-ascii"], ["profile1", "profile2"]) + + def test_non_ascii_course(self): + course = CourseFactory.create( + number=u"nón-äscii", + video_upload_pipeline={ + "course_video_upload_token": self.test_token, + } + ) + response = self.client.get(self.get_url_for_course_key(course.id)) + self.assertEqual(response.status_code, 200) + self.assertEqual( + response["Content-Disposition"], + "attachment; filename=video_urls.csv; filename*=utf-8''n%C3%B3n-%C3%A4scii_video_urls.csv" + ) diff --git a/cms/djangoapps/contentstore/views/videos.py b/cms/djangoapps/contentstore/views/videos.py index 8cb2ad4997..d3060d81aa 100644 --- a/cms/djangoapps/contentstore/views/videos.py +++ b/cms/djangoapps/contentstore/views/videos.py @@ -2,16 +2,20 @@ Views related to the video upload feature """ from boto import s3 +import csv from uuid import uuid4 from django.conf import settings from django.contrib.auth.decorators import login_required -from django.http import HttpResponseNotFound -from django.views.decorators.http import require_http_methods +from django.http import HttpResponse, HttpResponseNotFound +from django.utils.translation import ugettext as _ +from django.views.decorators.http import require_GET, require_http_methods +import rfc6266 from edxval.api import create_video, get_videos_for_ids from opaque_keys.edx.keys import CourseKey +from contentstore.models import VideoEncodingDownloadConfig from contentstore.utils import reverse_course_url from edxmako.shortcuts import render_to_response from util.json_request import expect_json, JsonResponse @@ -21,7 +25,7 @@ from xmodule.modulestore.django import modulestore from .course import get_course_and_check_access -__all__ = ["videos_handler"] +__all__ = ["videos_handler", "video_encodings_download"] # String constant used in asset keys to identify video assets. @@ -48,18 +52,9 @@ def videos_handler(request, course_key_string): to this endpoint but rather PUT to the respective upload_url values contained in the response """ - course_key = CourseKey.from_string(course_key_string) + course = _get_and_validate_course(course_key_string, request.user) - # For now, assume all studio users that have access to the course can upload videos. - # In the future, we plan to add a new org-level role for video uploaders. - course = get_course_and_check_access(course_key, request.user) - - if ( - not settings.FEATURES["ENABLE_VIDEO_UPLOAD_PIPELINE"] or - not getattr(settings, "VIDEO_UPLOAD_PIPELINE", None) or - not course or - not course.video_pipeline_configured - ): + if not course: return HttpResponseNotFound() if request.method == "GET": @@ -71,21 +66,132 @@ def videos_handler(request, course_key_string): return videos_post(course, request) +@login_required +@require_GET +def video_encodings_download(request, course_key_string): + """ + Returns a CSV report containing the encoded video URLs for video uploads + in the following format: + + Video ID,Name,Profile1 URL,Profile2 URL + aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa,video.mp4,http://example.com/profile1.mp4,http://example.com/profile2.mp4 + """ + course = _get_and_validate_course(course_key_string, request.user) + + if not course: + return HttpResponseNotFound() + + def get_profile_header(profile): + """Returns the column header string for the given profile's URLs""" + # Translators: This is the header for a CSV file column + # containing URLs for video encodings for the named profile + # (e.g. desktop, mobile high quality, mobile low quality) + return _("{profile_name} URL").format(profile_name=profile) + + profile_whitelist = VideoEncodingDownloadConfig.get_profile_whitelist() + status_whitelist = VideoEncodingDownloadConfig.get_status_whitelist() + + videos = list(_get_videos(course)) + name_col = _("Name") + duration_col = _("Duration") + added_col = _("Date Added") + video_id_col = _("Video ID") + profile_cols = [get_profile_header(profile) for profile in profile_whitelist] + + def make_csv_dict(video): + """ + Makes a dictionary suitable for writing CSV output. This involves + extracting the required items from the original video dict and + converting all keys and values to UTF-8 encoded string objects, + because the CSV module doesn't play well with unicode objects. + """ + # Translators: This is listed as the duration for a video that has not + # yet reached the point in its processing by the servers where its + # duration is determined. + duration_val = str(video["duration"]) if video["duration"] > 0 else _("Pending") + ret = dict( + [ + (name_col, video["client_video_id"]), + (duration_col, duration_val), + (added_col, video["created"].isoformat()), + (video_id_col, video["edx_video_id"]), + ] + + [ + (get_profile_header(encoded_video["profile"]), encoded_video["url"]) + for encoded_video in video["encoded_videos"] + if encoded_video["profile"] in profile_whitelist + ] + ) + return { + key.encode("utf-8"): value.encode("utf-8") + for key, value in ret.items() + } + + response = HttpResponse(content_type="text/csv") + # Translators: This is the suggested filename when downloading the URL + # listing for videos uploaded through Studio + filename = _("{course}_video_urls").format(course=course.id.course) + # See https://tools.ietf.org/html/rfc6266#appendix-D + response["Content-Disposition"] = rfc6266.build_header( + filename + ".csv", + filename_compat="video_urls.csv" + ) + writer = csv.DictWriter( + response, + [name_col, duration_col, added_col, video_id_col] + profile_cols, + dialect=csv.excel + ) + writer.writeheader() + for video in videos: + if video["status"] in status_whitelist: + writer.writerow(make_csv_dict(video)) + return response + + +def _get_and_validate_course(course_key_string, user): + """ + Given a course key, return the course if it exists, the given user has + access to it, and it is properly configured for video uploads + """ + course_key = CourseKey.from_string(course_key_string) + + # For now, assume all studio users that have access to the course can upload videos. + # In the future, we plan to add a new org-level role for video uploaders. + course = get_course_and_check_access(course_key, user) + + if ( + settings.FEATURES["ENABLE_VIDEO_UPLOAD_PIPELINE"] and + getattr(settings, "VIDEO_UPLOAD_PIPELINE", None) and + course and + course.video_pipeline_configured + ): + return course + else: + return None + + def _get_videos(course): """ Retrieves the list of videos from VAL corresponding to the videos listed in - the asset metadata store and returns the needed subset of fields + the asset metadata store """ edx_videos_ids = [ v.asset_id.path for v in modulestore().get_all_asset_metadata(course.id, VIDEO_ASSET_TYPE) ] + return get_videos_for_ids(edx_videos_ids) + + +def _get_index_videos(course): + """ + Returns the information about each video upload required for the video list + """ return list( { attr: video[attr] for attr in ["edx_video_id", "client_video_id", "created", "duration", "status"] } - for video in get_videos_for_ids(edx_videos_ids) + for video in _get_videos(course) ) @@ -98,7 +204,7 @@ def videos_index_html(course): { "context_course": course, "post_url": reverse_course_url("videos_handler", unicode(course.id)), - "previous_uploads": _get_videos(course), + "previous_uploads": _get_index_videos(course), "concurrent_upload_limit": settings.VIDEO_UPLOAD_PIPELINE.get("CONCURRENT_UPLOAD_LIMIT", 0), } ) @@ -117,7 +223,7 @@ def videos_index_json(course): }] } """ - return JsonResponse({"videos": _get_videos(course)}, status=200) + return JsonResponse({"videos": _get_index_videos(course)}, status=200) def videos_post(course, request): diff --git a/cms/urls.py b/cms/urls.py index 66d6964564..6017092a12 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -92,6 +92,7 @@ urlpatterns += patterns( url(r'^textbooks/{}$'.format(settings.COURSE_KEY_PATTERN), 'textbooks_list_handler'), url(r'^textbooks/{}/(?P\d[^/]*)$'.format(settings.COURSE_KEY_PATTERN), 'textbooks_detail_handler'), url(r'^videos/{}$'.format(settings.COURSE_KEY_PATTERN), 'videos_handler'), + url(r'^video_encodings_download/{}$'.format(settings.COURSE_KEY_PATTERN), 'video_encodings_download'), url(r'^group_configurations/{}$'.format(settings.COURSE_KEY_PATTERN), 'group_configurations_list_handler'), url(r'^group_configurations/{}/(?P\d+)/?$'.format(settings.COURSE_KEY_PATTERN), 'group_configurations_detail_handler'), diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 357d421c13..45d754167e 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -73,6 +73,7 @@ pysrt==0.4.7 PyYAML==3.10 requests==2.3.0 requests-oauthlib==0.4.1 +rfc6266==0.0.4 scipy==0.14.0 Shapely==1.2.16 singledispatch==3.4.0.2