Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add URL download endpoint for Studio video uploads

Browse Source 
The endpoint returns an Excel-dialect CSV file for download.
This commit is contained in:
Greg Price
2014-12-18 14:19:46 -05:00
committed by Nimisha Asthagiri
parent cc306843bc
commit 4e1925129e
8 changed files with 404 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cms/djangoapps/contentstore/admin.py Normal file
View File

@@ -0,0 +1,10 @@
"""
Admin site bindings for contentstore
"""
from django.contrib import admin
from config_models.admin import ConfigurationModelAdmin
from contentstore.models import VideoEncodingDownloadConfig
admin.site.register(VideoEncodingDownloadConfig, ConfigurationModelAdmin)

76
cms/djangoapps/contentstore/migrations/0001_initial.py Normal file
View File

@@ -0,0 +1,76 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'VideoEncodingDownloadConfig'
db.create_table('contentstore_videoencodingdownloadconfig', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('change_date', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
('changed_by', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'], null=True, on_delete=models.PROTECT)),
('enabled', self.gf('django.db.models.fields.BooleanField')(default=False)),
('profile_whitelist', self.gf('django.db.models.fields.TextField')(blank=True)),
('status_whitelist', self.gf('django.db.models.fields.TextField')(blank=True)),
))
db.send_create_signal('contentstore', ['VideoEncodingDownloadConfig'])
def backwards(self, orm):
# Deleting model 'VideoEncodingDownloadConfig'
db.delete_table('contentstore_videoencodingdownloadconfig')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contentstore.videoencodingdownloadconfig': {
'Meta': {'object_name': 'VideoEncodingDownloadConfig'},
'change_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'changed_by': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'on_delete': 'models.PROTECT'}),
'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'profile_whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'status_whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
}
}
complete_apps = ['contentstore']

0
cms/djangoapps/contentstore/migrations/__init__.py Normal file
View File

32
cms/djangoapps/contentstore/models.py Normal file
View File

@@ -0,0 +1,32 @@
"""
Models for contentstore
"""
from django.db.models.fields import TextField
from config_models.models import ConfigurationModel
class VideoEncodingDownloadConfig(ConfigurationModel):
"""Configuration for what to include in video encoding downloads"""
profile_whitelist = TextField(
blank=True,
help_text="A comma-separated list of names of profiles to include in video encoding downloads"
)
status_whitelist = TextField(
blank=True,
help_text="A comma-separated list of status values; only videos with these status values will be included in video encoding downloads"
)
@classmethod
def get_profile_whitelist(cls):
"""Get the list of profiles to include in the encoding download"""
return [profile for profile in cls.current().profile_whitelist.split(",") if profile]
@classmethod
def get_status_whitelist(cls):
"""
Get the list of status values to include files for in the encoding
download
"""
return [status for status in cls.current().status_whitelist.split(",") if status]

178
cms/djangoapps/contentstore/views/tests/test_videos.py
View File

@@ -1,43 +1,59 @@
#-*- coding: utf-8 -*-
"""
Unit tests for video-related REST APIs.
"""
# pylint: disable=attribute-defined-outside-init
import csv
import json
import dateutil.parser
import re
from StringIO import StringIO
from django.conf import settings
from django.test.utils import override_settings
from mock import Mock, patch
from edxval.api import create_video, get_video_info
from edxval.api import create_profile, create_video, get_video_info
from contentstore.models import VideoEncodingDownloadConfig
from contentstore.views.videos import KEY_EXPIRATION_IN_SECONDS, VIDEO_ASSET_TYPE
from contentstore.tests.utils import CourseTestCase
from contentstore.utils import reverse_course_url
from xmodule.assetstore import AssetMetadata
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.factories import CourseFactory
@patch.dict("django.conf.settings.FEATURES", {"ENABLE_VIDEO_UPLOAD_PIPELINE": True})
@override_settings(VIDEO_UPLOAD_PIPELINE={"BUCKET": "test_bucket", "ROOT_PATH": "test_root"})
class VideoUploadTestCase(CourseTestCase):
class VideoUploadTestMixin(object):
"""
Test cases for the video upload page
Test cases for the video upload feature
"""
@staticmethod
def get_url_for_course_key(course_key):
def get_url_for_course_key(self, course_key):
"""Return video handler URL for the given course"""
return reverse_course_url("videos_handler", course_key)
return reverse_course_url(self.VIEW_NAME, course_key)
def setUp(self):
super(VideoUploadTestCase, self).setUp()
self.url = VideoUploadTestCase.get_url_for_course_key(self.course.id)
super(VideoUploadTestMixin, self).setUp()
self.url = self.get_url_for_course_key(self.course.id)
self.test_token = "test_token"
self.course.video_upload_pipeline = {
"course_video_upload_token": self.test_token,
}
self.save_course()
self.profiles = [
{
"profile_name": "profile1",
"extension": "mp4",
"width": 640,
"height": 480,
},
{
"profile_name": "profile2",
"extension": "mp4",
"width": 1920,
"height": 1080,
},
]
self.previous_uploads = [
{
"edx_video_id": "test1",
@@ -51,9 +67,38 @@ class VideoUploadTestCase(CourseTestCase):
"client_video_id": "test2.mp4",
"duration": 128.0,
"status": "file_complete",
"encoded_videos": [],
}
"encoded_videos": [
{
"profile": "profile1",
"url": "http://example.com/profile1/test2.mp4",
"file_size": 1600,
"bitrate": 100,
},
{
"profile": "profile2",
"url": "http://example.com/profile2/test2.mov",
"file_size": 16000,
"bitrate": 1000,
},
],
},
{
"edx_video_id": "non-ascii",
"client_video_id": u"nón-ascii-näme.mp4",
"duration": 256.0,
"status": "file_delivered",
"encoded_videos": [
{
"profile": "profile1",
"url": u"http://example.com/profile1/nón-ascii-näme.mp4",
"file_size": 3200,
"bitrate": 100,
},
]
},
]
for profile in self.profiles:
create_profile(profile)
for video in self.previous_uploads:
create_video(video)
modulestore().save_asset_metadata(
@@ -63,6 +108,14 @@ class VideoUploadTestCase(CourseTestCase):
self.user.id
)
def _get_previous_upload(self, edx_video_id):
"""Returns the previous upload with the given video id."""
return next(
video
for video in self.previous_uploads
if video["edx_video_id"] == edx_video_id
)
def test_anon_user(self):
self.client.logout()
response = self.client.get(self.url)
@@ -74,7 +127,7 @@ class VideoUploadTestCase(CourseTestCase):
def test_invalid_course_key(self):
response = self.client.get(
VideoUploadTestCase.get_url_for_course_key("Non/Existent/Course")
self.get_url_for_course_key("Non/Existent/Course")
)
self.assertEqual(response.status_code, 404)
@@ -96,17 +149,21 @@ class VideoUploadTestCase(CourseTestCase):
self.save_course()
self.assertEqual(self.client.get(self.url).status_code, 404)
@patch.dict("django.conf.settings.FEATURES", {"ENABLE_VIDEO_UPLOAD_PIPELINE": True})
@override_settings(VIDEO_UPLOAD_PIPELINE={"BUCKET": "test_bucket", "ROOT_PATH": "test_root"})
class VideosHandlerTestCase(VideoUploadTestMixin, CourseTestCase):
"""Test cases for the main video upload endpoint"""
VIEW_NAME = "videos_handler"
def test_get_json(self):
response = self.client.get_json(self.url)
self.assertEqual(response.status_code, 200)
response_videos = json.loads(response.content)["videos"]
self.assertEqual(len(response_videos), len(self.previous_uploads))
for response_video in response_videos:
original_video = dict(
next(
video for video in self.previous_uploads if video["edx_video_id"] == response_video["edx_video_id"]
)
)
original_video = self._get_previous_upload(response_video["edx_video_id"])
self.assertEqual(
set(response_video.keys()),
set(["edx_video_id", "client_video_id", "created", "duration", "status"])
@@ -191,6 +248,7 @@ class VideoUploadTestCase(CourseTestCase):
json.dumps({"files": files}),
content_type="application/json"
)
self.assertEqual(response.status_code, 200)
response_obj = json.loads(response.content)
mock_conn.assert_called_once_with(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
@@ -243,3 +301,87 @@ class VideoUploadTestCase(CourseTestCase):
response_file = response_obj["files"][i]
self.assertEqual(response_file["file_name"], file_info["file_name"])
self.assertEqual(response_file["upload_url"], mock_key_instance.generate_url())
@patch.dict("django.conf.settings.FEATURES", {"ENABLE_VIDEO_UPLOAD_PIPELINE": True})
@override_settings(VIDEO_UPLOAD_PIPELINE={"BUCKET": "test_bucket", "ROOT_PATH": "test_root"})
class VideoUrlsCsvTestCase(VideoUploadTestMixin, CourseTestCase):
"""Test cases for the CSV download endpoint for video uploads"""
VIEW_NAME = "video_encodings_download"
def setUp(self):
super(VideoUrlsCsvTestCase, self).setUp()
VideoEncodingDownloadConfig(
profile_whitelist="profile1",
status_whitelist="file_delivered,file_complete"
).save()
def _check_csv_response(self, expected_video_ids, expected_profiles):
"""
Check that the response is a valid CSV response containing rows
corresponding to expected_video_ids.
"""
response = self.client.get(self.url)
self.assertEqual(response.status_code, 200)
self.assertEqual(
response["Content-Disposition"],
"attachment; filename={course}_video_urls.csv".format(course=self.course.id.course)
)
response_reader = StringIO(response.content)
reader = csv.DictReader(response_reader, dialect=csv.excel)
self.assertEqual(
reader.fieldnames,
(
["Name", "Duration", "Date Added", "Video ID"] +
["{} URL".format(profile) for profile in expected_profiles]
)
)
actual_video_ids = []
for row in reader:
response_video = {
key.decode("utf-8"): value.decode("utf-8") for key, value in row.items()
}
actual_video_ids.append(response_video["Video ID"])
original_video = self._get_previous_upload(response_video["Video ID"])
self.assertEqual(response_video["Name"], original_video["client_video_id"])
self.assertEqual(response_video["Video ID"], original_video["edx_video_id"])
for profile in expected_profiles:
response_profile_url = response_video["{} URL".format(profile)]
original_encoded_for_profile = next(
(
original_encoded
for original_encoded in original_video["encoded_videos"]
if original_encoded["profile"] == profile
),
None
)
if original_encoded_for_profile:
self.assertEqual(response_profile_url, original_encoded_for_profile["url"])
else:
self.assertEqual(response_profile_url, "")
self.assertEqual(set(actual_video_ids), set(expected_video_ids))
def test_basic(self):
self._check_csv_response(["test2", "non-ascii"], ["profile1"])
def test_config(self):
VideoEncodingDownloadConfig(
profile_whitelist="profile1,profile2",
status_whitelist="file_delivered,file_complete,transcode_active"
).save()
self._check_csv_response(["test1", "test2", "non-ascii"], ["profile1", "profile2"])
def test_non_ascii_course(self):
course = CourseFactory.create(
number=u"nón-äscii",
video_upload_pipeline={
"course_video_upload_token": self.test_token,
}
)
response = self.client.get(self.get_url_for_course_key(course.id))
self.assertEqual(response.status_code, 200)
self.assertEqual(
response["Content-Disposition"],
"attachment; filename=video_urls.csv; filename*=utf-8''n%C3%B3n-%C3%A4scii_video_urls.csv"
)

142
cms/djangoapps/contentstore/views/videos.py
View File

@@ -2,16 +2,20 @@
Views related to the video upload feature
"""
from boto import s3
import csv
from uuid import uuid4
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.http import HttpResponseNotFound
from django.views.decorators.http import require_http_methods
from django.http import HttpResponse, HttpResponseNotFound
from django.utils.translation import ugettext as _
from django.views.decorators.http import require_GET, require_http_methods
import rfc6266
from edxval.api import create_video, get_videos_for_ids
from opaque_keys.edx.keys import CourseKey
from contentstore.models import VideoEncodingDownloadConfig
from contentstore.utils import reverse_course_url
from edxmako.shortcuts import render_to_response
from util.json_request import expect_json, JsonResponse
@@ -21,7 +25,7 @@ from xmodule.modulestore.django import modulestore
from .course import get_course_and_check_access
__all__ = ["videos_handler"]
__all__ = ["videos_handler", "video_encodings_download"]
# String constant used in asset keys to identify video assets.
@@ -48,18 +52,9 @@ def videos_handler(request, course_key_string):
to this endpoint but rather PUT to the respective upload_url values
contained in the response
"""
course_key = CourseKey.from_string(course_key_string)
course = _get_and_validate_course(course_key_string, request.user)
# For now, assume all studio users that have access to the course can upload videos.
# In the future, we plan to add a new org-level role for video uploaders.
course = get_course_and_check_access(course_key, request.user)
if (
not settings.FEATURES["ENABLE_VIDEO_UPLOAD_PIPELINE"] or
not getattr(settings, "VIDEO_UPLOAD_PIPELINE", None) or
not course or
not course.video_pipeline_configured
):
if not course:
return HttpResponseNotFound()
if request.method == "GET":
@@ -71,21 +66,132 @@ def videos_handler(request, course_key_string):
return videos_post(course, request)
@login_required
@require_GET
def video_encodings_download(request, course_key_string):
"""
Returns a CSV report containing the encoded video URLs for video uploads
in the following format:
Video ID,Name,Profile1 URL,Profile2 URL
aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa,video.mp4,http://example.com/profile1.mp4,http://example.com/profile2.mp4
"""
course = _get_and_validate_course(course_key_string, request.user)
if not course:
return HttpResponseNotFound()
def get_profile_header(profile):
"""Returns the column header string for the given profile's URLs"""
# Translators: This is the header for a CSV file column
# containing URLs for video encodings for the named profile
# (e.g. desktop, mobile high quality, mobile low quality)
return _("{profile_name} URL").format(profile_name=profile)
profile_whitelist = VideoEncodingDownloadConfig.get_profile_whitelist()
status_whitelist = VideoEncodingDownloadConfig.get_status_whitelist()
videos = list(_get_videos(course))
name_col = _("Name")
duration_col = _("Duration")
added_col = _("Date Added")
video_id_col = _("Video ID")
profile_cols = [get_profile_header(profile) for profile in profile_whitelist]
def make_csv_dict(video):
"""
Makes a dictionary suitable for writing CSV output. This involves
extracting the required items from the original video dict and
converting all keys and values to UTF-8 encoded string objects,
because the CSV module doesn't play well with unicode objects.
"""
# Translators: This is listed as the duration for a video that has not
# yet reached the point in its processing by the servers where its
# duration is determined.
duration_val = str(video["duration"]) if video["duration"] > 0 else _("Pending")
ret = dict(
[
(name_col, video["client_video_id"]),
(duration_col, duration_val),
(added_col, video["created"].isoformat()),
(video_id_col, video["edx_video_id"]),
] +
[
(get_profile_header(encoded_video["profile"]), encoded_video["url"])
for encoded_video in video["encoded_videos"]
if encoded_video["profile"] in profile_whitelist
]
)
return {
key.encode("utf-8"): value.encode("utf-8")
for key, value in ret.items()
}
response = HttpResponse(content_type="text/csv")
# Translators: This is the suggested filename when downloading the URL
# listing for videos uploaded through Studio
filename = _("{course}_video_urls").format(course=course.id.course)
# See https://tools.ietf.org/html/rfc6266#appendix-D
response["Content-Disposition"] = rfc6266.build_header(
filename + ".csv",
filename_compat="video_urls.csv"
)
writer = csv.DictWriter(
response,
[name_col, duration_col, added_col, video_id_col] + profile_cols,
dialect=csv.excel
)
writer.writeheader()
for video in videos:
if video["status"] in status_whitelist:
writer.writerow(make_csv_dict(video))
return response
def _get_and_validate_course(course_key_string, user):
"""
Given a course key, return the course if it exists, the given user has
access to it, and it is properly configured for video uploads
"""
course_key = CourseKey.from_string(course_key_string)
# For now, assume all studio users that have access to the course can upload videos.
# In the future, we plan to add a new org-level role for video uploaders.
course = get_course_and_check_access(course_key, user)
if (
settings.FEATURES["ENABLE_VIDEO_UPLOAD_PIPELINE"] and
getattr(settings, "VIDEO_UPLOAD_PIPELINE", None) and
course and
course.video_pipeline_configured
):
return course
else:
return None
def _get_videos(course):
"""
Retrieves the list of videos from VAL corresponding to the videos listed in
the asset metadata store and returns the needed subset of fields
the asset metadata store
"""
edx_videos_ids = [
v.asset_id.path
for v in modulestore().get_all_asset_metadata(course.id, VIDEO_ASSET_TYPE)
]
return get_videos_for_ids(edx_videos_ids)
def _get_index_videos(course):
"""
Returns the information about each video upload required for the video list
"""
return list(
{
attr: video[attr]
for attr in ["edx_video_id", "client_video_id", "created", "duration", "status"]
}
for video in get_videos_for_ids(edx_videos_ids)
for video in _get_videos(course)
)
@@ -98,7 +204,7 @@ def videos_index_html(course):
{
"context_course": course,
"post_url": reverse_course_url("videos_handler", unicode(course.id)),
"previous_uploads": _get_videos(course),
"previous_uploads": _get_index_videos(course),
"concurrent_upload_limit": settings.VIDEO_UPLOAD_PIPELINE.get("CONCURRENT_UPLOAD_LIMIT", 0),
}
)
@@ -117,7 +223,7 @@ def videos_index_json(course):
}]
}
"""
return JsonResponse({"videos": _get_videos(course)}, status=200)
return JsonResponse({"videos": _get_index_videos(course)}, status=200)
def videos_post(course, request):

1
cms/urls.py
View File

@@ -92,6 +92,7 @@ urlpatterns += patterns(
url(r'^textbooks/{}$'.format(settings.COURSE_KEY_PATTERN), 'textbooks_list_handler'),
url(r'^textbooks/{}/(?P<textbook_id>\d[^/]*)$'.format(settings.COURSE_KEY_PATTERN), 'textbooks_detail_handler'),
url(r'^videos/{}$'.format(settings.COURSE_KEY_PATTERN), 'videos_handler'),
url(r'^video_encodings_download/{}$'.format(settings.COURSE_KEY_PATTERN), 'video_encodings_download'),
url(r'^group_configurations/{}$'.format(settings.COURSE_KEY_PATTERN), 'group_configurations_list_handler'),
url(r'^group_configurations/{}/(?P<group_configuration_id>\d+)/?$'.format(settings.COURSE_KEY_PATTERN),
'group_configurations_detail_handler'),

1
requirements/edx/base.txt
View File

@@ -73,6 +73,7 @@ pysrt==0.4.7
PyYAML==3.10
requests==2.3.0
requests-oauthlib==0.4.1
rfc6266==0.0.4
scipy==0.14.0
Shapely==1.2.16
singledispatch==3.4.0.2