Files
edx-platform/cms/djangoapps/contentstore/views/assets.py
Nathan Sprenkle c97932fa99 fix: add missing protocol to web link for assets (#27220)
* fix: add missing protocol to web link for assets

* test: fix asset path test

* refactor: update asset web URL to use urljoin
2021-04-02 11:13:57 -04:00

609 lines
22 KiB
Python

"""Views for assets"""
import json
import logging
import math
import re
from functools import partial
from urllib.parse import urljoin
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.core.exceptions import PermissionDenied
from django.http import HttpResponseBadRequest, HttpResponseNotFound
from django.utils.translation import ugettext as _
from django.views.decorators.csrf import ensure_csrf_cookie
from django.views.decorators.http import require_http_methods, require_POST
from opaque_keys.edx.keys import AssetKey, CourseKey
from pymongo import ASCENDING, DESCENDING
from common.djangoapps.edxmako.shortcuts import render_to_response
from common.djangoapps.student.auth import has_course_author_access
from common.djangoapps.util.date_utils import get_default_time_display
from common.djangoapps.util.json_request import JsonResponse
from openedx.core.djangoapps.contentserver.caching import del_cached_content
from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
from xmodule.exceptions import NotFoundError
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from ..utils import reverse_course_url
from .exception import AssetNotFoundException, AssetSizeTooLargeException
__all__ = ['assets_handler']
REQUEST_DEFAULTS = {
'page': 0,
'page_size': 50,
'sort': 'date_added',
'direction': '',
'asset_type': '',
'text_search': '',
}
@login_required
@ensure_csrf_cookie
def assets_handler(request, course_key_string=None, asset_key_string=None):
'''
The restful handler for assets.
It allows retrieval of all the assets (as an HTML page), as well as uploading new assets,
deleting assets, and changing the 'locked' state of an asset.
GET
html: return an html page which will show all course assets. Note that only the asset container
is returned and that the actual assets are filled in with a client-side request.
json: returns a page of assets. The following parameters are supported:
page: the desired page of results (defaults to 0)
page_size: the number of items per page (defaults to 50)
sort: the asset field to sort by (defaults to 'date_added')
direction: the sort direction (defaults to 'descending')
asset_type: the file type to filter items to (defaults to All)
text_search: string to filter results by file name (defaults to '')
POST
json: create (or update?) an asset. The only updating that can be done is changing the lock state.
PUT
json: update the locked state of an asset
DELETE
json: delete an asset
'''
course_key = CourseKey.from_string(course_key_string)
if not has_course_author_access(request.user, course_key):
raise PermissionDenied()
response_format = _get_response_format(request)
if _request_response_format_is_json(request, response_format):
if request.method == 'GET':
return _assets_json(request, course_key)
asset_key = AssetKey.from_string(asset_key_string) if asset_key_string else None
return _update_asset(request, course_key, asset_key)
elif request.method == 'GET': # assume html
return _asset_index(request, course_key)
return HttpResponseNotFound()
def _get_response_format(request):
return request.GET.get('format') or request.POST.get('format') or 'html'
def _request_response_format_is_json(request, response_format):
return response_format == 'json' or 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json')
def _asset_index(request, course_key):
'''
Display an editable asset library.
Supports start (0-based index into the list of assets) and max query parameters.
'''
course_module = modulestore().get_course(course_key)
return render_to_response('asset_index.html', {
'language_code': request.LANGUAGE_CODE,
'context_course': course_module,
'max_file_size_in_mbs': settings.MAX_ASSET_UPLOAD_FILE_SIZE_IN_MB,
'chunk_size_in_mbs': settings.UPLOAD_CHUNK_SIZE_IN_MB,
'max_file_size_redirect_url': settings.MAX_ASSET_UPLOAD_FILE_SIZE_URL,
'asset_callback_url': reverse_course_url('assets_handler', course_key)
})
def _assets_json(request, course_key):
'''
Display an editable asset library.
Supports start (0-based index into the list of assets) and max query parameters.
'''
request_options = _parse_request_to_dictionary(request)
filter_parameters = {}
if request_options['requested_asset_type']:
filters_are_invalid_error = _get_error_if_invalid_parameters(request_options['requested_asset_type'])
if filters_are_invalid_error is not None:
return filters_are_invalid_error
filter_parameters.update(_get_content_type_filter_for_mongo(request_options['requested_asset_type']))
if request_options['requested_text_search']:
filter_parameters.update(_get_displayname_search_filter_for_mongo(request_options['requested_text_search']))
sort_type_and_direction = _get_sort_type_and_direction(request_options)
requested_page_size = request_options['requested_page_size']
current_page = _get_current_page(request_options['requested_page'])
first_asset_to_display_index = _get_first_asset_index(current_page, requested_page_size)
query_options = {
'current_page': current_page,
'page_size': requested_page_size,
'sort': sort_type_and_direction,
'filter_params': filter_parameters
}
assets, total_count = _get_assets_for_page(course_key, query_options)
if request_options['requested_page'] > 0 and first_asset_to_display_index >= total_count and total_count > 0: # lint-amnesty, pylint: disable=chained-comparison
_update_options_to_requery_final_page(query_options, total_count)
current_page = query_options['current_page']
first_asset_to_display_index = _get_first_asset_index(current_page, requested_page_size)
assets, total_count = _get_assets_for_page(course_key, query_options)
last_asset_to_display_index = first_asset_to_display_index + len(assets)
assets_in_json_format = _get_assets_in_json_format(assets, course_key)
response_payload = {
'start': first_asset_to_display_index,
'end': last_asset_to_display_index,
'page': current_page,
'pageSize': requested_page_size,
'totalCount': total_count,
'assets': assets_in_json_format,
'sort': request_options['requested_sort'],
'direction': request_options['requested_sort_direction'],
'assetTypes': _get_requested_file_types_from_requested_filter(request_options['requested_asset_type']),
'textSearch': request_options['requested_text_search'],
}
return JsonResponse(response_payload)
def _parse_request_to_dictionary(request):
return {
'requested_page': int(_get_requested_attribute(request, 'page')),
'requested_page_size': int(_get_requested_attribute(request, 'page_size')),
'requested_sort': _get_requested_attribute(request, 'sort'),
'requested_sort_direction': _get_requested_attribute(request, 'direction'),
'requested_asset_type': _get_requested_attribute(request, 'asset_type'),
'requested_text_search': _get_requested_attribute(request, 'text_search'),
}
def _get_requested_attribute(request, attribute):
return request.GET.get(attribute, REQUEST_DEFAULTS.get(attribute))
def _get_error_if_invalid_parameters(requested_filter):
"""Function for returning error messages on filters"""
requested_file_types = _get_requested_file_types_from_requested_filter(requested_filter)
invalid_filters = []
# OTHER is not described in the settings file as a filter
all_valid_file_types = set(_get_files_and_upload_type_filters().keys())
all_valid_file_types.add('OTHER')
for requested_file_type in requested_file_types:
if requested_file_type not in all_valid_file_types:
invalid_filters.append(requested_file_type)
if invalid_filters:
error_message = {
'error_code': 'invalid_asset_type_filter',
'developer_message': 'The asset_type parameter to the request is invalid. '
'The {} filters are not described in the settings.FILES_AND_UPLOAD_TYPE_FILTERS '
'dictionary.'.format(invalid_filters)
}
return JsonResponse({'error': error_message}, status=400)
def _get_content_type_filter_for_mongo(requested_filter):
"""
Construct and return pymongo query dict for the given content type categories.
"""
requested_file_types = _get_requested_file_types_from_requested_filter(requested_filter)
type_filter = {
"$or": []
}
if 'OTHER' in requested_file_types:
type_filter["$or"].append(_get_mongo_expression_for_type_other())
requested_file_types.remove('OTHER')
type_filter["$or"].append(_get_mongo_expression_for_type_filter(requested_file_types))
return type_filter
def _get_mongo_expression_for_type_other():
"""
Construct and return pymongo expression dict for the 'OTHER' content type category.
"""
content_types = [ext for extensions in _get_files_and_upload_type_filters().values() for ext in extensions]
return {
'contentType': {
'$nin': content_types
}
}
def _get_mongo_expression_for_type_filter(requested_file_types):
"""
Construct and return pymongo expression dict for the named content type categories.
The named content categories are the keys of the FILES_AND_UPLOAD_TYPE_FILTERS setting that are not 'OTHER':
'Images', 'Documents', 'Audio', and 'Code'.
"""
content_types = []
files_and_upload_type_filters = _get_files_and_upload_type_filters()
for requested_file_type in requested_file_types:
content_types.extend(files_and_upload_type_filters[requested_file_type])
return {
'contentType': {
'$in': content_types
}
}
def _get_displayname_search_filter_for_mongo(text_search):
"""
Return a pymongo query dict for the given search string, using case insensitivity.
"""
filters = []
text_search_tokens = text_search.split()
for token in text_search_tokens:
escaped_token = re.escape(token)
filters.append({
'displayname': {
'$regex': escaped_token,
'$options': 'i',
},
})
return {
'$and': filters,
}
def _get_files_and_upload_type_filters():
return settings.FILES_AND_UPLOAD_TYPE_FILTERS
def _get_requested_file_types_from_requested_filter(requested_filter):
return requested_filter.split(',') if requested_filter else []
def _get_sort_type_and_direction(request_options):
sort_type = _get_mongo_sort_from_requested_sort(request_options['requested_sort'])
sort_direction = _get_sort_direction_from_requested_sort(request_options['requested_sort_direction'])
return [(sort_type, sort_direction)]
def _get_mongo_sort_from_requested_sort(requested_sort):
"""Function returns sorts dataset based on the key provided"""
if requested_sort == 'date_added':
sort = 'uploadDate'
elif requested_sort == 'display_name':
sort = 'displayname'
else:
sort = requested_sort
return sort
def _get_sort_direction_from_requested_sort(requested_sort_direction):
if requested_sort_direction.lower() == 'asc':
return ASCENDING
return DESCENDING
def _get_current_page(requested_page):
return max(requested_page, 0)
def _get_first_asset_index(current_page, page_size):
return current_page * page_size
def _get_assets_for_page(course_key, options):
"""returns course content for given course and options"""
current_page = options['current_page']
page_size = options['page_size']
sort = options['sort']
filter_params = options['filter_params'] if options['filter_params'] else None
start = current_page * page_size
return contentstore().get_all_content_for_course(
course_key, start=start, maxresults=page_size, sort=sort, filter_params=filter_params
)
def _update_options_to_requery_final_page(query_options, total_asset_count):
"""sets current_page value based on asset count and page_size"""
query_options['current_page'] = int(math.floor((total_asset_count - 1) / query_options['page_size']))
def _get_assets_in_json_format(assets, course_key):
"""returns assets information in JSON Format"""
assets_in_json_format = []
for asset in assets:
thumbnail_asset_key = _get_thumbnail_asset_key(asset, course_key)
asset_is_locked = asset.get('locked', False)
asset_in_json = _get_asset_json(
asset['displayname'],
asset['contentType'],
asset['uploadDate'],
asset['asset_key'],
thumbnail_asset_key,
asset_is_locked
)
assets_in_json_format.append(asset_in_json)
return assets_in_json_format
def update_course_run_asset(course_key, upload_file):
"""returns contents of the uploaded file"""
course_exists_response = _get_error_if_course_does_not_exist(course_key)
if course_exists_response is not None:
return course_exists_response
file_metadata = _get_file_metadata_as_dictionary(upload_file)
is_file_too_large = _check_file_size_is_too_large(file_metadata)
if is_file_too_large:
error_message = _get_file_too_large_error_message(file_metadata['filename'])
raise AssetSizeTooLargeException(error_message)
content, temporary_file_path = _get_file_content_and_path(file_metadata, course_key)
(thumbnail_content, thumbnail_location) = contentstore().generate_thumbnail(content,
tempfile_path=temporary_file_path)
# delete cached thumbnail even if one couldn't be created this time (else the old thumbnail will continue to show)
del_cached_content(thumbnail_location)
if _check_thumbnail_uploaded(thumbnail_content):
content.thumbnail_location = thumbnail_location
contentstore().save(content)
del_cached_content(content.location)
return content
@require_POST
@ensure_csrf_cookie
@login_required
def _upload_asset(request, course_key):
"""uploads the file in request and returns JSON response"""
course_exists_error = _get_error_if_course_does_not_exist(course_key)
if course_exists_error is not None:
return course_exists_error
# compute a 'filename' which is similar to the location formatting, we're
# using the 'filename' nomenclature since we're using a FileSystem paradigm
# here. We're just imposing the Location string formatting expectations to
# keep things a bit more consistent
upload_file = request.FILES['file']
try:
content = update_course_run_asset(course_key, upload_file)
except AssetSizeTooLargeException as exception:
return JsonResponse({'error': str(exception)}, status=413)
# readback the saved content - we need the database timestamp
readback = contentstore().find(content.location)
locked = getattr(content, 'locked', False)
return JsonResponse({
'asset': _get_asset_json(
content.name,
content.content_type,
readback.last_modified_at,
content.location,
content.thumbnail_location,
locked
),
'msg': _('Upload completed')
})
def _get_error_if_course_does_not_exist(course_key): # lint-amnesty, pylint: disable=missing-function-docstring
try:
modulestore().get_course(course_key)
except ItemNotFoundError:
logging.error('Could not find course: %s', course_key)
return HttpResponseBadRequest()
def _get_file_metadata_as_dictionary(upload_file): # lint-amnesty, pylint: disable=missing-function-docstring
# compute a 'filename' which is similar to the location formatting; we're
# using the 'filename' nomenclature since we're using a FileSystem paradigm
# here; we're just imposing the Location string formatting expectations to
# keep things a bit more consistent
return {
'upload_file': upload_file,
'filename': upload_file.name,
'mime_type': upload_file.content_type,
'upload_file_size': get_file_size(upload_file)
}
def get_file_size(upload_file):
"""returns the size of the uploaded file"""
# can be used for mocking test file sizes.
return upload_file.size
def _check_file_size_is_too_large(file_metadata):
"""verifies whether file size is greater than allowed file size"""
upload_file_size = file_metadata['upload_file_size']
maximum_file_size_in_megabytes = settings.MAX_ASSET_UPLOAD_FILE_SIZE_IN_MB
maximum_file_size_in_bytes = maximum_file_size_in_megabytes * 1000 ** 2
return upload_file_size > maximum_file_size_in_bytes
def _get_file_too_large_error_message(filename):
"""returns formatted error message for large files"""
return _(
'File {filename} exceeds maximum size of '
'{maximum_size_in_megabytes} MB.'
).format(
filename=filename,
maximum_size_in_megabytes=settings.MAX_ASSET_UPLOAD_FILE_SIZE_IN_MB,
)
def _get_file_content_and_path(file_metadata, course_key):
"""returns contents of the uploaded file and path for temporary uploaded file"""
content_location = StaticContent.compute_location(course_key, file_metadata['filename'])
upload_file = file_metadata['upload_file']
file_can_be_chunked = upload_file.multiple_chunks()
static_content_partial = partial(StaticContent, content_location, file_metadata['filename'],
file_metadata['mime_type'])
if file_can_be_chunked:
content = static_content_partial(upload_file.chunks())
temporary_file_path = upload_file.temporary_file_path()
else:
content = static_content_partial(upload_file.read())
temporary_file_path = None
return content, temporary_file_path
def _check_thumbnail_uploaded(thumbnail_content):
"""returns whether thumbnail is None"""
return thumbnail_content is not None
def _get_thumbnail_asset_key(asset, course_key):
"""returns thumbnail asset key"""
# note, due to the schema change we may not have a 'thumbnail_location' in the result set
thumbnail_location = asset.get('thumbnail_location', None)
thumbnail_asset_key = None
if thumbnail_location:
thumbnail_path = thumbnail_location[4]
thumbnail_asset_key = course_key.make_asset_key('thumbnail', thumbnail_path)
return thumbnail_asset_key
@require_http_methods(('DELETE', 'POST', 'PUT'))
@login_required
@ensure_csrf_cookie
def _update_asset(request, course_key, asset_key):
"""
restful CRUD operations for a course asset.
Currently only DELETE, POST, and PUT methods are implemented.
asset_path_encoding: the odd /c4x/org/course/category/name repr of the asset (used by Backbone as the id)
"""
if request.method == 'DELETE':
try:
delete_asset(course_key, asset_key)
return JsonResponse()
except AssetNotFoundException:
return JsonResponse(status=404)
elif request.method in ('PUT', 'POST'):
if 'file' in request.FILES:
return _upload_asset(request, course_key)
# update existing asset
try:
modified_asset = json.loads(request.body.decode('utf8'))
except ValueError:
return HttpResponseBadRequest()
contentstore().set_attr(asset_key, 'locked', modified_asset['locked'])
# delete the asset from the cache so we check the lock status the next time it is requested.
del_cached_content(asset_key)
return JsonResponse(modified_asset, status=201)
def _save_content_to_trash(content):
"""saves the content to trash"""
contentstore('trashcan').save(content)
def delete_asset(course_key, asset_key):
"""deletes the cached content based on asset key"""
content = _check_existence_and_get_asset_content(asset_key)
_save_content_to_trash(content)
_delete_thumbnail(content.thumbnail_location, course_key, asset_key)
contentstore().delete(content.get_id())
del_cached_content(content.location)
def _check_existence_and_get_asset_content(asset_key): # lint-amnesty, pylint: disable=missing-function-docstring
try:
content = contentstore().find(asset_key)
return content
except NotFoundError:
raise AssetNotFoundException # lint-amnesty, pylint: disable=raise-missing-from
def _delete_thumbnail(thumbnail_location, course_key, asset_key): # lint-amnesty, pylint: disable=missing-function-docstring
if thumbnail_location is not None:
# We are ignoring the value of the thumbnail_location-- we only care whether
# or not a thumbnail has been stored, and we can now easily create the correct path.
thumbnail_location = course_key.make_asset_key('thumbnail', asset_key.block_id)
try:
thumbnail_content = contentstore().find(thumbnail_location)
_save_content_to_trash(thumbnail_content)
contentstore().delete(thumbnail_content.get_id())
del_cached_content(thumbnail_location)
except Exception: # pylint: disable=broad-except
logging.warning('Could not delete thumbnail: %s', thumbnail_location)
def _get_asset_json(display_name, content_type, date, location, thumbnail_location, locked):
'''
Helper method for formatting the asset information to send to client.
'''
asset_url = StaticContent.serialize_asset_key_with_slash(location)
external_url = urljoin(configuration_helpers.get_value('LMS_ROOT_URL', settings.LMS_ROOT_URL), asset_url)
return {
'display_name': display_name,
'content_type': content_type,
'date_added': get_default_time_display(date),
'url': asset_url,
'external_url': external_url,
'portable_url': StaticContent.get_static_path_from_location(location),
'thumbnail': StaticContent.serialize_asset_key_with_slash(thumbnail_location) if thumbnail_location else None,
'locked': locked,
# needed for Backbone delete/update.
'id': str(location)
}