From ad773dac5cf10308156617889d273ed7d282e8a7 Mon Sep 17 00:00:00 2001 From: Tyler Hallada Date: Fri, 5 Jan 2018 17:48:31 -0500 Subject: [PATCH] Add text_search param for searching by displayname --- cms/djangoapps/contentstore/views/assets.py | 125 ++++++++++-------- .../contentstore/views/tests/test_assets.py | 27 ++++ cms/envs/common.py | 2 +- 3 files changed, 98 insertions(+), 56 deletions(-) diff --git a/cms/djangoapps/contentstore/views/assets.py b/cms/djangoapps/contentstore/views/assets.py index 3abd21e556..2ffd504464 100644 --- a/cms/djangoapps/contentstore/views/assets.py +++ b/cms/djangoapps/contentstore/views/assets.py @@ -2,6 +2,7 @@ import json import logging import math from functools import partial +import re from django.conf import settings from django.contrib.auth.decorators import login_required @@ -34,7 +35,8 @@ REQUEST_DEFAULTS = { 'page_size': 50, 'sort': 'date_added', 'direction': '', - 'asset_type': '' + 'asset_type': '', + 'text_search': '', } @@ -54,6 +56,8 @@ def assets_handler(request, course_key_string=None, asset_key_string=None): page_size: the number of items per page (defaults to 50) sort: the asset field to sort by (defaults to 'date_added') direction: the sort direction (defaults to 'descending') + asset_type: the file type to filter items to (defaults to All) + text_search: string to filter results by file name (defaults to '') POST json: create (or update?) an asset. The only updating that can be done is changing the lock state. PUT @@ -113,7 +117,7 @@ def _assets_json(request, course_key): ''' request_options = _parse_request_to_dictionary(request) - filter_parameters = None + filter_parameters = {} if request_options['requested_asset_type']: filters_are_invalid_error = _get_error_if_invalid_parameters(request_options['requested_asset_type']) @@ -121,7 +125,10 @@ def _assets_json(request, course_key): if filters_are_invalid_error is not None: return filters_are_invalid_error - filter_parameters = _get_filter_parameters_for_mongo(request_options['requested_asset_type']) + filter_parameters.update(_get_content_type_filter_for_mongo(request_options['requested_asset_type'])) + + if request_options['requested_text_search']: + filter_parameters.update(_get_displayname_search_filter_for_mongo(request_options['requested_text_search'])) sort_type_and_direction = _get_sort_type_and_direction(request_options) @@ -157,6 +164,7 @@ def _assets_json(request, course_key): 'sort': request_options['requested_sort'], 'direction': request_options['requested_sort_direction'], 'assetTypes': _get_requested_file_types_from_requested_filter(request_options['requested_asset_type']), + 'textSearch': request_options['requested_text_search'], } return JsonResponse(response_payload) @@ -169,6 +177,7 @@ def _parse_request_to_dictionary(request): 'requested_sort': _get_requested_attribute(request, 'sort'), 'requested_sort_direction': _get_requested_attribute(request, 'direction'), 'requested_asset_type': _get_requested_attribute(request, 'asset_type'), + 'requested_text_search': _get_requested_attribute(request, 'text_search'), } @@ -198,57 +207,77 @@ def _get_error_if_invalid_parameters(requested_filter): return JsonResponse({'error': error_message}, status=400) -def _get_filter_parameters_for_mongo(requested_filter): +def _get_content_type_filter_for_mongo(requested_filter): + """ + Construct and return pymongo query dict for the given content type categories. + """ requested_file_types = _get_requested_file_types_from_requested_filter(requested_filter) - mongo_where_operator_parameters = _get_mongo_where_operator_parameters_for_filters(requested_file_types) + type_filter = { + "$or": [] + } - return mongo_where_operator_parameters + if 'OTHER' in requested_file_types: + type_filter["$or"].append(_get_mongo_expression_for_type_other()) + requested_file_types.remove('OTHER') + + type_filter["$or"].append(_get_mongo_expression_for_type_filter(requested_file_types)) + + return type_filter -def _get_mongo_where_operator_parameters_for_filters(requested_file_types): - javascript_filters = [] - - for requested_file_type in requested_file_types: - if requested_file_type == 'OTHER': - javascript_filters_for_file_types = _get_javascript_expressions_for_other_() - javascript_filters.append(javascript_filters_for_file_types) - else: - javascript_filters_for_file_types = _get_javascript_expressions_for_filter(requested_file_type) - javascript_filters.append(javascript_filters_for_file_types) - - javascript_filters = _join_javascript_expressions_for_filters_with_separator(javascript_filters, '||') - - return _format_javascript_filters_for_mongo_where(javascript_filters) - - -def _format_javascript_filters_for_mongo_where(javascript_filters): +def _get_mongo_expression_for_type_other(): + """ + Construct and return pymongo expression dict for the 'OTHER' content type category. + """ + content_types = [ext for extensions in _get_files_and_upload_type_filters().values() for ext in extensions] return { - '$where': javascript_filters, + 'contentType': { + '$nin': content_types + } } -def _get_javascript_expressions_for_other_(): - file_extensions_for_requested_file_types = _get_files_and_upload_type_filters().values() - file_extensions_for_requested_file_types_flattened = [extension for extensions in - file_extensions_for_requested_file_types for extension in - extensions] +def _get_mongo_expression_for_type_filter(requested_file_types): + """ + Construct and return pymongo expression dict for the named content type categories. - javascript_expression_to_filter_extensions = _get_javascript_expressions_to_filter_extensions_with_operator( - file_extensions_for_requested_file_types_flattened, '!=') - joined_javascript_expressions_to_filter_extensions = _join_javascript_expressions_for_filters_with_separator( - javascript_expression_to_filter_extensions, ' && ') + The named content categories are the keys of the FILES_AND_UPLOAD_TYPE_FILTERS setting that are not 'OTHER': + 'Images', 'Documents', 'Audio', and 'Code'. + """ + content_types = [] + files_and_upload_type_filters = _get_files_and_upload_type_filters() - return joined_javascript_expressions_to_filter_extensions + for requested_file_type in requested_file_types: + content_types.extend(files_and_upload_type_filters[requested_file_type]) + + return { + 'contentType': { + '$in': content_types + } + } -def _get_javascript_expressions_for_filter(requested_file_type): - file_extensions_for_requested_file_type = _get_extensions_for_file_type(requested_file_type) - javascript_expressions_to_filter_extensions = _get_javascript_expressions_to_filter_extensions_with_operator( - file_extensions_for_requested_file_type, '==') - joined_javascript_expressions_to_filter_extensions = _join_javascript_expressions_for_filters_with_separator( - javascript_expressions_to_filter_extensions, ' || ') +def _get_displayname_search_filter_for_mongo(text_search): + """ + Return a pymongo query dict for the given search string, using case insensitivity. + """ + filters = [] - return joined_javascript_expressions_to_filter_extensions + text_search_tokens = text_search.split() + + for token in text_search_tokens: + escaped_token = re.escape(token) + + filters.append({ + 'displayname': { + '$regex': escaped_token, + '$options': 'i', + }, + }) + + return { + '$and': filters, + } def _get_files_and_upload_type_filters(): @@ -259,19 +288,6 @@ def _get_requested_file_types_from_requested_filter(requested_filter): return requested_filter.split(',') if requested_filter else [] -def _get_extensions_for_file_type(requested_file_type): - return _get_files_and_upload_type_filters().get(requested_file_type) - - -def _get_javascript_expressions_to_filter_extensions_with_operator(file_extensions, operator): - return ["JSON.stringify(this.contentType).toUpperCase() " + operator + " JSON.stringify('{}').toUpperCase()".format( - file_extension) for file_extension in file_extensions] - - -def _join_javascript_expressions_for_filters_with_separator(javascript_expressions_for_filtering, separator): - return separator.join(javascript_expressions_for_filtering) - - def _get_sort_type_and_direction(request_options): sort_type = _get_mongo_sort_from_requested_sort(request_options['requested_sort']) sort_direction = _get_sort_direction_from_requested_sort(request_options['requested_sort_direction']) @@ -309,7 +325,6 @@ def _get_assets_for_page(course_key, options): sort = options['sort'] filter_params = options['filter_params'] if options['filter_params'] else None start = current_page * page_size - return contentstore().get_all_content_for_course( course_key, start=start, maxresults=page_size, sort=sort, filter_params=filter_params ) diff --git a/cms/djangoapps/contentstore/views/tests/test_assets.py b/cms/djangoapps/contentstore/views/tests/test_assets.py index 0eb64734ca..d1e911d07f 100644 --- a/cms/djangoapps/contentstore/views/tests/test_assets.py +++ b/cms/djangoapps/contentstore/views/tests/test_assets.py @@ -179,6 +179,16 @@ class PaginationTestCase(AssetsTestCase): self.assert_correct_filter_response( self.url, 'asset_type', 'Documents,OTHER') + self.assert_correct_text_search_response(self.url, 'asset-1.txt', 1) + self.assert_correct_text_search_response(self.url, 'asset-1', 1) + self.assert_correct_text_search_response(self.url, 'AsSeT-1', 1) + self.assert_correct_text_search_response(self.url, '.txt', 3) + self.assert_correct_text_search_response(self.url, '2', 1) + self.assert_correct_text_search_response(self.url, 'asset 2', 1) + self.assert_correct_text_search_response(self.url, '2 asset', 1) + self.assert_correct_text_search_response(self.url, '*.txt', 0) + self.assert_correct_asset_response(self.url + "?text_search=", 0, 4, 4) + #Verify invalid request parameters self.assert_invalid_parameters_error(self.url, 'asset_type', 'edX') self.assert_invalid_parameters_error(self.url, 'asset_type', 'edX, OTHER') @@ -300,6 +310,23 @@ class PaginationTestCase(AssetsTestCase): url + '?' + filter_type + '=' + filter_value, HTTP_ACCEPT='application/json') self.assertEquals(resp.status_code, 400) + def assert_correct_text_search_response(self, url, text_search, number_matches): + """ + Get from the url w/ a text_search option and ensure items honor that search query + """ + resp = self.client.get( + url + '?text_search=' + text_search, HTTP_ACCEPT='application/json') + json_response = json.loads(resp.content) + assets_response = json_response['assets'] + self.assertEquals(text_search, json_response['textSearch']) + self.assertEquals(len(assets_response), number_matches) + + text_search_tokens = text_search.split() + + for asset_response in assets_response: + for token in text_search_tokens: + self.assertTrue(token.lower() in asset_response['display_name'].lower()) + @ddt class UploadTestCase(AssetsTestCase): diff --git a/cms/envs/common.py b/cms/envs/common.py index f7d289a7d2..e18517ba36 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -1331,7 +1331,7 @@ FILES_AND_UPLOAD_TYPE_FILTERS = { 'text/csv', 'text/pdf', 'text/x-sh', - '\application/pdf\""', + '\"application/pdf\"', ], "Audio": ['audio/mpeg', 'audio/mp3', 'audio/x-wav', 'audio/ogg', 'audio/wav', 'audio/aac', 'audio/x-m4a', 'audio/mp4', 'audio/x-ms-wma', ],