Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #6314 from edx/release

Browse Source 
Release
This commit is contained in:
Fred Smith
2014-12-19 13:23:22 -05:00
parent 1cf5aa0c16 dbc6166f7e
commit 251407d7d2
348 changed files with 54006 additions and 56882 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cms/djangoapps/contentstore/tests/utils.py
View File

@@ -94,7 +94,7 @@ class CourseTestCase(ModuleStoreTestCase):
"""
nonstaff, password = self.create_non_staff_user()
client = Client()
client = AjaxEnabledTestClient()
if authenticate:
client.login(username=nonstaff.username, password=password)
nonstaff.is_authenticated = True

1
cms/djangoapps/contentstore/views/__init__.py
View File

@@ -17,6 +17,7 @@ from .public import *
from .export_git import *
from .user import *
from .tabs import *
from .videos import *
from .transcripts_ajax import *
try:
from .dev import *

24
cms/djangoapps/contentstore/views/course.py
View File

@@ -89,7 +89,7 @@ class AccessListFallback(Exception):
pass
def _get_course_module(course_key, user, depth=0):
def get_course_and_check_access(course_key, user, depth=0):
"""
Internal method used to calculate and return the locator and course module
for the view functions in this file.
@@ -214,7 +214,7 @@ def course_handler(request, course_key_string=None):
if request.method == 'GET':
course_key = CourseKey.from_string(course_key_string)
with modulestore().bulk_operations(course_key):
course_module = _get_course_module(course_key, request.user, depth=None)
course_module = get_course_and_check_access(course_key, request.user, depth=None)
return JsonResponse(_course_outline_json(request, course_module))
elif request.method == 'POST': # not sure if this is only post. If one will have ids, it goes after access
return _create_or_rerun_course(request)
@@ -251,7 +251,7 @@ def course_rerun_handler(request, course_key_string):
raise PermissionDenied()
course_key = CourseKey.from_string(course_key_string)
with modulestore().bulk_operations(course_key):
course_module = _get_course_module(course_key, request.user, depth=3)
course_module = get_course_and_check_access(course_key, request.user, depth=3)
if request.method == 'GET':
return render_to_response('course-create-rerun.html', {
'source_course_key': course_key,
@@ -434,7 +434,7 @@ def course_index(request, course_key):
# A depth of None implies the whole course. The course outline needs this in order to compute has_changes.
# A unit may not have a draft version, but one of its components could, and hence the unit itself has changes.
with modulestore().bulk_operations(course_key):
course_module = _get_course_module(course_key, request.user, depth=None)
course_module = get_course_and_check_access(course_key, request.user, depth=None)
lms_link = get_lms_link_for_item(course_module.location)
sections = course_module.get_children()
course_structure = _course_outline_json(request, course_module)
@@ -662,7 +662,7 @@ def course_info_handler(request, course_key_string):
"""
course_key = CourseKey.from_string(course_key_string)
with modulestore().bulk_operations(course_key):
course_module = _get_course_module(course_key, request.user)
course_module = get_course_and_check_access(course_key, request.user)
if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'):
return render_to_response(
'course_info.html',
@@ -745,7 +745,7 @@ def settings_handler(request, course_key_string):
"""
course_key = CourseKey.from_string(course_key_string)
with modulestore().bulk_operations(course_key):
course_module = _get_course_module(course_key, request.user)
course_module = get_course_and_check_access(course_key, request.user)
if 'text/html' in request.META.get('HTTP_ACCEPT', '') and request.method == 'GET':
upload_asset_url = reverse_course_url('assets_handler', course_key)
@@ -800,7 +800,7 @@ def grading_handler(request, course_key_string, grader_index=None):
"""
course_key = CourseKey.from_string(course_key_string)
with modulestore().bulk_operations(course_key):
course_module = _get_course_module(course_key, request.user)
course_module = get_course_and_check_access(course_key, request.user)
if 'text/html' in request.META.get('HTTP_ACCEPT', '') and request.method == 'GET':
course_details = CourseGradingModel.fetch(course_key)
@@ -912,7 +912,7 @@ def advanced_settings_handler(request, course_key_string):
"""
course_key = CourseKey.from_string(course_key_string)
with modulestore().bulk_operations(course_key):
course_module = _get_course_module(course_key, request.user)
course_module = get_course_and_check_access(course_key, request.user)
if 'text/html' in request.META.get('HTTP_ACCEPT', '') and request.method == 'GET':
return render_to_response('settings_advanced.html', {
@@ -1026,7 +1026,7 @@ def textbooks_list_handler(request, course_key_string):
course_key = CourseKey.from_string(course_key_string)
store = modulestore()
with store.bulk_operations(course_key):
course = _get_course_module(course_key, request.user)
course = get_course_and_check_access(course_key, request.user)
if "application/json" not in request.META.get('HTTP_ACCEPT', 'text/html'):
# return HTML page
@@ -1102,7 +1102,7 @@ def textbooks_detail_handler(request, course_key_string, textbook_id):
course_key = CourseKey.from_string(course_key_string)
store = modulestore()
with store.bulk_operations(course_key):
course_module = _get_course_module(course_key, request.user)
course_module = get_course_and_check_access(course_key, request.user)
matching_id = [tb for tb in course_module.pdf_textbooks
if unicode(tb.get("id")) == unicode(textbook_id)]
if matching_id:
@@ -1333,7 +1333,7 @@ def group_configurations_list_handler(request, course_key_string):
course_key = CourseKey.from_string(course_key_string)
store = modulestore()
with store.bulk_operations(course_key):
course = _get_course_module(course_key, request.user)
course = get_course_and_check_access(course_key, request.user)
if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'):
group_configuration_url = reverse_course_url('group_configurations_list_handler', course_key)
@@ -1381,7 +1381,7 @@ def group_configurations_detail_handler(request, course_key_string, group_config
course_key = CourseKey.from_string(course_key_string)
store = modulestore()
with store.bulk_operations(course_key):
course = _get_course_module(course_key, request.user)
course = get_course_and_check_access(course_key, request.user)
matching_id = [p for p in course.user_partitions
if unicode(p.id) == unicode(group_configuration_id)]
if matching_id:

245
cms/djangoapps/contentstore/views/tests/test_videos.py Normal file
View File

@@ -0,0 +1,245 @@
"""
Unit tests for video-related REST APIs.
"""
# pylint: disable=attribute-defined-outside-init
import json
import dateutil.parser
import re
from django.conf import settings
from django.test.utils import override_settings
from mock import Mock, patch
from edxval.api import create_video, get_video_info
from contentstore.views.videos import KEY_EXPIRATION_IN_SECONDS, VIDEO_ASSET_TYPE
from contentstore.tests.utils import CourseTestCase
from contentstore.utils import reverse_course_url
from xmodule.assetstore import AssetMetadata
from xmodule.modulestore.django import modulestore
@patch.dict("django.conf.settings.FEATURES", {"ENABLE_VIDEO_UPLOAD_PIPELINE": True})
@override_settings(VIDEO_UPLOAD_PIPELINE={"BUCKET": "test_bucket", "ROOT_PATH": "test_root"})
class VideoUploadTestCase(CourseTestCase):
"""
Test cases for the video upload page
"""
@staticmethod
def get_url_for_course_key(course_key):
"""Return video handler URL for the given course"""
return reverse_course_url("videos_handler", course_key)
def setUp(self):
super(VideoUploadTestCase, self).setUp()
self.url = VideoUploadTestCase.get_url_for_course_key(self.course.id)
self.test_token = "test_token"
self.course.video_upload_pipeline = {
"course_video_upload_token": self.test_token,
}
self.save_course()
self.previous_uploads = [
{
"edx_video_id": "test1",
"client_video_id": "test1.mp4",
"duration": 42.0,
"status": "transcode_active",
"encoded_videos": [],
},
{
"edx_video_id": "test2",
"client_video_id": "test2.mp4",
"duration": 128.0,
"status": "file_complete",
"encoded_videos": [],
}
]
for video in self.previous_uploads:
create_video(video)
modulestore().save_asset_metadata(
AssetMetadata(
self.course.id.make_asset_key(VIDEO_ASSET_TYPE, video["edx_video_id"])
),
self.user.id
)
def test_anon_user(self):
self.client.logout()
response = self.client.get(self.url)
self.assertEqual(response.status_code, 302)
def test_put(self):
response = self.client.put(self.url)
self.assertEqual(response.status_code, 405)
def test_invalid_course_key(self):
response = self.client.get(
VideoUploadTestCase.get_url_for_course_key("Non/Existent/Course")
)
self.assertEqual(response.status_code, 404)
def test_non_staff_user(self):
client, __ = self.create_non_staff_authed_user_client()
response = client.get(self.url)
self.assertEqual(response.status_code, 403)
def test_video_pipeline_not_enabled(self):
settings.FEATURES["ENABLE_VIDEO_UPLOAD_PIPELINE"] = False
self.assertEqual(self.client.get(self.url).status_code, 404)
def test_video_pipeline_not_configured(self):
settings.VIDEO_UPLOAD_PIPELINE = None
self.assertEqual(self.client.get(self.url).status_code, 404)
def test_course_not_configured(self):
self.course.video_upload_pipeline = {}
self.save_course()
self.assertEqual(self.client.get(self.url).status_code, 404)
def test_get_json(self):
response = self.client.get_json(self.url)
self.assertEqual(response.status_code, 200)
response_videos = json.loads(response.content)["videos"]
self.assertEqual(len(response_videos), len(self.previous_uploads))
for response_video in response_videos:
original_video = dict(
next(
video for video in self.previous_uploads if video["edx_video_id"] == response_video["edx_video_id"]
)
)
self.assertEqual(
set(response_video.keys()),
set(["edx_video_id", "client_video_id", "created", "duration", "status"])
)
dateutil.parser.parse(response_video["created"])
for field in ["edx_video_id", "client_video_id", "duration", "status"]:
self.assertEqual(response_video[field], original_video[field])
def test_get_html(self):
response = self.client.get(self.url)
self.assertEqual(response.status_code, 200)
self.assertRegexpMatches(response["Content-Type"], "^text/html(;.*)?$")
# Crude check for presence of data in returned HTML
for video in self.previous_uploads:
self.assertIn(video["edx_video_id"], response.content)
def test_post_non_json(self):
response = self.client.post(self.url, {"files": []})
self.assertEqual(response.status_code, 400)
def test_post_malformed_json(self):
response = self.client.post(self.url, "{", content_type="application/json")
self.assertEqual(response.status_code, 400)
def test_post_invalid_json(self):
def assert_bad(content):
"""Make request with content and assert that response is 400"""
response = self.client.post(
self.url,
json.dumps(content),
content_type="application/json"
)
self.assertEqual(response.status_code, 400)
# Top level missing files key
assert_bad({})
# Entry missing file_name
assert_bad({"files": [{"content_type": "video/mp4"}]})
# Entry missing content_type
assert_bad({"files": [{"file_name": "test.mp4"}]})
@override_settings(AWS_ACCESS_KEY_ID="test_key_id", AWS_SECRET_ACCESS_KEY="test_secret")
@patch("boto.s3.key.Key")
@patch("boto.s3.connection.S3Connection")
def test_post_success(self, mock_conn, mock_key):
files = [
{
"file_name": "first.mp4",
"content_type": "video/mp4",
},
{
"file_name": "second.webm",
"content_type": "video/webm",
},
{
"file_name": "third.mov",
"content_type": "video/quicktime",
},
{
"file_name": "fourth.mp4",
"content_type": "video/mp4",
},
]
bucket = Mock()
mock_conn.return_value = Mock(get_bucket=Mock(return_value=bucket))
mock_key_instances = [
Mock(
generate_url=Mock(
return_value="http://example.com/url_{}".format(file_info["file_name"])
)
)
for file_info in files
]
# If extra calls are made, return a dummy
mock_key.side_effect = mock_key_instances + [Mock()]
response = self.client.post(
self.url,
json.dumps({"files": files}),
content_type="application/json"
)
response_obj = json.loads(response.content)
mock_conn.assert_called_once_with(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
self.assertEqual(len(response_obj["files"]), len(files))
self.assertEqual(mock_key.call_count, len(files))
for i, file_info in enumerate(files):
# Ensure Key was set up correctly and extract id
key_call_args, __ = mock_key.call_args_list[i]
self.assertEqual(key_call_args[0], bucket)
path_match = re.match(
(
settings.VIDEO_UPLOAD_PIPELINE["ROOT_PATH"] +
"/([a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12})$"
),
key_call_args[1]
)
self.assertIsNotNone(path_match)
video_id = path_match.group(1)
mock_key_instance = mock_key_instances[i]
mock_key_instance.set_metadata.assert_any_call(
"course_video_upload_token",
self.test_token
)
mock_key_instance.set_metadata.assert_any_call(
"client_video_id",
file_info["file_name"]
)
mock_key_instance.set_metadata.assert_any_call("course_key", unicode(self.course.id))
mock_key_instance.generate_url.assert_called_once_with(
KEY_EXPIRATION_IN_SECONDS,
"PUT",
headers={"Content-Type": file_info["content_type"]}
)
# Ensure asset store was updated
self.assertIsNotNone(
modulestore().find_asset_metadata(
self.course.id.make_asset_key(VIDEO_ASSET_TYPE, video_id)
)
)
# Ensure VAL was updated
val_info = get_video_info(video_id)
self.assertEqual(val_info["status"], "upload")
self.assertEqual(val_info["client_video_id"], file_info["file_name"])
self.assertEqual(val_info["status"], "upload")
self.assertEqual(val_info["duration"], 0)
# Ensure response is correct
response_file = response_obj["files"][i]
self.assertEqual(response_file["file_name"], file_info["file_name"])
self.assertEqual(response_file["upload_url"], mock_key_instance.generate_url())

214
cms/djangoapps/contentstore/views/videos.py Normal file
View File

@@ -0,0 +1,214 @@
"""
Views related to the video upload feature
"""
from boto import s3
from uuid import uuid4
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.http import HttpResponseNotFound
from django.views.decorators.http import require_http_methods
from edxval.api import create_video, get_videos_for_ids
from opaque_keys.edx.keys import CourseKey
from contentstore.utils import reverse_course_url
from edxmako.shortcuts import render_to_response
from util.json_request import expect_json, JsonResponse
from xmodule.assetstore import AssetMetadata
from xmodule.modulestore.django import modulestore
from .course import get_course_and_check_access
__all__ = ["videos_handler"]
# String constant used in asset keys to identify video assets.
VIDEO_ASSET_TYPE = "video"
# Default expiration, in seconds, of one-time URLs used for uploading videos.
KEY_EXPIRATION_IN_SECONDS = 86400
@expect_json
@login_required
@require_http_methods(("GET", "POST"))
def videos_handler(request, course_key_string):
"""
The restful handler for video uploads.
GET
html: return an HTML page to display previous video uploads and allow
new ones
json: return json representing the videos that have been uploaded and
their statuses
POST
json: create a new video upload; the actual files should not be provided
to this endpoint but rather PUT to the respective upload_url values
contained in the response
"""
course_key = CourseKey.from_string(course_key_string)
# For now, assume all studio users that have access to the course can upload videos.
# In the future, we plan to add a new org-level role for video uploaders.
course = get_course_and_check_access(course_key, request.user)
if (
not settings.FEATURES["ENABLE_VIDEO_UPLOAD_PIPELINE"] or
not getattr(settings, "VIDEO_UPLOAD_PIPELINE", None) or
not course or
not course.video_pipeline_configured
):
return HttpResponseNotFound()
if request.method == "GET":
if "application/json" in request.META.get("HTTP_ACCEPT", ""):
return videos_index_json(course)
else:
return videos_index_html(course)
else:
return videos_post(course, request)
def _get_videos(course):
"""
Retrieves the list of videos from VAL corresponding to the videos listed in
the asset metadata store and returns the needed subset of fields
"""
edx_videos_ids = [
v.asset_id.path
for v in modulestore().get_all_asset_metadata(course.id, VIDEO_ASSET_TYPE)
]
return list(
{
attr: video[attr]
for attr in ["edx_video_id", "client_video_id", "created", "duration", "status"]
}
for video in get_videos_for_ids(edx_videos_ids)
)
def videos_index_html(course):
"""
Returns an HTML page to display previous video uploads and allow new ones
"""
return render_to_response(
"videos_index.html",
{
"context_course": course,
"post_url": reverse_course_url("videos_handler", unicode(course.id)),
"previous_uploads": _get_videos(course),
"concurrent_upload_limit": settings.VIDEO_UPLOAD_PIPELINE.get("CONCURRENT_UPLOAD_LIMIT", 0),
}
)
def videos_index_json(course):
"""
Returns JSON in the following format:
{
"videos": [{
"edx_video_id": "aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa",
"client_video_id": "video.mp4",
"created": "1970-01-01T00:00:00Z",
"duration": 42.5,
"status": "upload"
}]
}
"""
return JsonResponse({"videos": _get_videos(course)}, status=200)
def videos_post(course, request):
"""
Input (JSON):
{
"files": [{
"file_name": "video.mp4",
"content_type": "video/mp4"
}]
}
Returns (JSON):
{
"files": [{
"file_name": "video.mp4",
"upload_url": "http://example.com/put_video"
}]
}
The returned array corresponds exactly to the input array.
"""
error = None
if "files" not in request.json:
error = "Request object is not JSON or does not contain 'files'"
elif any(
"file_name" not in file or "content_type" not in file
for file in request.json["files"]
):
error = "Request 'files' entry does not contain 'file_name' and 'content_type'"
if error:
return JsonResponse({"error": error}, status=400)
bucket = storage_service_bucket()
course_video_upload_token = course.video_upload_pipeline["course_video_upload_token"]
req_files = request.json["files"]
resp_files = []
for req_file in req_files:
file_name = req_file["file_name"]
edx_video_id = unicode(uuid4())
key = storage_service_key(bucket, file_name=edx_video_id)
for metadata_name, value in [
("course_video_upload_token", course_video_upload_token),
("client_video_id", file_name),
("course_key", unicode(course.id)),
]:
key.set_metadata(metadata_name, value)
upload_url = key.generate_url(
KEY_EXPIRATION_IN_SECONDS,
"PUT",
headers={"Content-Type": req_file["content_type"]}
)
# persist edx_video_id as uploaded through this course
video_meta_data = AssetMetadata(course.id.make_asset_key(VIDEO_ASSET_TYPE, edx_video_id))
modulestore().save_asset_metadata(video_meta_data, request.user.id)
# persist edx_video_id in VAL
create_video({
"edx_video_id": edx_video_id,
"status": "upload",
"client_video_id": file_name,
"duration": 0,
"encoded_videos": [],
})
resp_files.append({"file_name": file_name, "upload_url": upload_url})
return JsonResponse({"files": resp_files}, status=200)
def storage_service_bucket():
"""
Returns an S3 bucket for video uploads.
"""
conn = s3.connection.S3Connection(
settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY
)
return conn.get_bucket(settings.VIDEO_UPLOAD_PIPELINE["BUCKET"])
def storage_service_key(bucket, file_name):
"""
Returns an S3 key to the given file in the given bucket.
"""
key_name = "{}/{}".format(
settings.VIDEO_UPLOAD_PIPELINE.get("ROOT_PATH", ""),
file_name
)
return s3.key.Key(bucket, key_name)