Merge pull request #1325 from edx/dhm/paginate_assets

Add asset pagination

cms/djangoapps/contentstore/tests/test_assets.py

@@ -6,19 +6,20 @@ Unit tests for the asset upload endpoint.
 #pylint: disable=W0621
 #pylint: disable=W0212
 
-from datetime import datetime
+from datetime import datetime, timedelta
 from io import BytesIO
 from pytz import UTC
+import json
+import re
 from unittest import TestCase, skip
 from .utils import CourseTestCase
 from django.core.urlresolvers import reverse
 from contentstore.views import assets
-from xmodule.contentstore.content import StaticContent
+from xmodule.contentstore.content import StaticContent, XASSET_LOCATION_TAG
 from xmodule.modulestore import Location
 from xmodule.contentstore.django import contentstore
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.xml_importer import import_from_xml
-import json
 
 
 class AssetsTestCase(CourseTestCase):
@@ -147,3 +148,84 @@ class LockAssetTestCase(CourseTestCase):
         resp_asset = post_asset_update(False)
         self.assertFalse(resp_asset['locked'])
         verify_asset_locked_state(False)
+
+
+class TestAssetIndex(CourseTestCase):
+    """
+    Test getting asset lists via http (Note, the assets don't actually exist)
+    """
+    def setUp(self):
+        """
+        Create fake asset entries for the other tests to use
+        """
+        super(TestAssetIndex, self).setUp()
+        self.entry_filter = self.create_asset_entries(contentstore(), 100)
+
+    def tearDown(self):
+        """
+        Get rid of the entries
+        """
+        contentstore().fs_files.remove(self.entry_filter)
+
+    def create_asset_entries(self, cstore, number):
+        """
+        Create the fake entries
+        """
+        course_filter = Location(
+            XASSET_LOCATION_TAG, category='asset', course=self.course.location.course, org=self.course.location.org
+        )
+        base_entry = {
+            'displayname': 'foo.jpg',
+            'chunkSize': 262144,
+            'length': 0,
+            'uploadDate': datetime(2012, 1, 2, 0, 0),
+            'contentType': 'image/jpeg',
+        }
+        for i in range(number):
+            base_entry['displayname'] = '{:03x}.jpeg'.format(i)
+            base_entry['uploadDate'] += timedelta(hours=i)
+            base_entry['_id'] = course_filter.replace(name=base_entry['displayname']).dict()
+            cstore.fs_files.insert(base_entry)
+
+        return course_filter.dict()
+
+    ASSET_LIST_RE = re.compile(r'AssetCollection\((.*)\);$', re.MULTILINE)
+
+    def check_page_content(self, resp_content, entry_count, last_date=None):
+        """
+        :param entry_count:
+        :param last_date:
+        """
+        match = self.ASSET_LIST_RE.search(resp_content)
+        asset_list = json.loads(match.group(1))
+        self.assertEqual(len(asset_list), entry_count)
+        for row in asset_list:
+            datetext = row['date_added']
+            parsed_date = datetime.strptime(datetext, "%b %d, %Y at %H:%M UTC")
+            if last_date is None:
+                last_date = parsed_date
+            else:
+                self.assertGreaterEqual(last_date, parsed_date)
+        return last_date
+
+    def test_query_assets(self):
+        """
+        The actual test
+        """
+        # get all
+        asset_url = reverse(
+            'asset_index',
+            kwargs={
+                'org': self.course.location.org,
+                'course': self.course.location.course,
+                'name': self.course.location.name
+            }
+        )
+        resp = self.client.get(asset_url)
+        self.check_page_content(resp.content, 100)
+        # get first page of 10
+        resp = self.client.get(asset_url + "/max/10")
+        last_date = self.check_page_content(resp.content, 10)
+        # get next of 20
+        resp = self.client.get(asset_url + "/start/10/max/20")
+        last_date = self.check_page_content(resp.content, 20, last_date)

cms/djangoapps/contentstore/views/assets.py

@@ -23,6 +23,7 @@ from .access import get_location_and_verify_access
 from util.json_request import JsonResponse
 import json
 from django.utils.translation import ugettext as _
+from pymongo import DESCENDING
 
 
 __all__ = ['asset_index', 'upload_asset']
@@ -30,11 +31,14 @@ __all__ = ['asset_index', 'upload_asset']
 
 @login_required
 @ensure_csrf_cookie
-def asset_index(request, org, course, name):
+def asset_index(request, org, course, name, start=None, maxresults=None):
     """
     Display an editable asset library
 
     org, course, name: Attributes of the Location for the item to edit
+
+    :param start: which index of the result list to start w/, used for paging results
+    :param maxresults: maximum results
     """
     location = get_location_and_verify_access(request, org, course, name)
 
@@ -47,10 +51,17 @@ def asset_index(request, org, course, name):
     course_module = modulestore().get_item(location)
 
     course_reference = StaticContent.compute_location(org, course, name)
-    assets = contentstore().get_all_content_for_course(course_reference)
-
-    # sort in reverse upload date order
-    assets = sorted(assets, key=lambda asset: asset['uploadDate'], reverse=True)
+    if maxresults is not None:
+        maxresults = int(maxresults)
+        start = int(start) if start else 0
+        assets = contentstore().get_all_content_for_course(
+            course_reference, start=start, maxresults=maxresults,
+            sort=[('uploadDate', DESCENDING)]
+        )
+    else:
+        assets = contentstore().get_all_content_for_course(
+            course_reference, sort=[('uploadDate', DESCENDING)]
+        )
 
     asset_json = []
     for asset in assets:

cms/urls.py

@@ -71,7 +71,7 @@ urlpatterns = patterns('',  # nopep8
     url(r'^edit_tabs/(?P<org>[^/]+)/(?P<course>[^/]+)/course/(?P<coursename>[^/]+)$',
         'contentstore.views.edit_tabs', name='edit_tabs'),
 
-    url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/assets/(?P<name>[^/]+)$',
+    url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/assets/(?P<name>[^/]+)(/start/(?P<start>\d+))?(/max/(?P<maxresults>\d+))?$',
         'contentstore.views.asset_index', name='asset_index'),
     url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/assets/(?P<name>[^/]+)/(?P<asset_id>.+)?.*$',
         'contentstore.views.assets.update_asset', name='update_asset'),

common/lib/xmodule/xmodule/contentstore/content.py

@@ -168,7 +168,7 @@ class ContentStore(object):
     def find(self, filename):
         raise NotImplementedError
 
-    def get_all_content_for_course(self, location):
+    def get_all_content_for_course(self, location, start=0, maxresults=-1, sort=None):
         '''
         Returns a list of all static assets for a course. The return format is a list of dictionary elements. Example:
 

common/lib/xmodule/xmodule/contentstore/mongo.py

@@ -130,10 +130,12 @@ class MongoContentStore(ContentStore):
     def get_all_content_thumbnails_for_course(self, location):
         return self._get_all_content_for_course(location, get_thumbnails=True)
 
-    def get_all_content_for_course(self, location):
-        return self._get_all_content_for_course(location, get_thumbnails=False)
+    def get_all_content_for_course(self, location, start=0, maxresults=-1, sort=None):
+        return self._get_all_content_for_course(
+            location, start=start, maxresults=maxresults, get_thumbnails=False, sort=sort
+        )
 
-    def _get_all_content_for_course(self, location, get_thumbnails=False):
+    def _get_all_content_for_course(self, location, get_thumbnails=False, start=0, maxresults=-1, sort=None):
         '''
         Returns a list of all static assets for a course. The return format is a list of dictionary elements. Example:
 
@@ -156,7 +158,13 @@ class MongoContentStore(ContentStore):
         course_filter = Location(XASSET_LOCATION_TAG, category="asset" if not get_thumbnails else "thumbnail",
                                  course=location.course, org=location.org)
         # 'borrow' the function 'location_to_query' from the Mongo modulestore implementation
-        items = self.fs_files.find(location_to_query(course_filter))
+        if maxresults > 0:
+            items = self.fs_files.find(
+                location_to_query(course_filter),
+                skip=start, limit=maxresults, sort=sort
+            )
+        else:
+            items = self.fs_files.find(location_to_query(course_filter), sort=sort)
         return list(items)
 
     def set_attr(self, location, attr, value=True):

lms/djangoapps/courseware/courses.py

@@ -106,24 +106,24 @@ def course_image_url(course):
     if course.static_asset_path or modulestore().get_modulestore_type(course.location.course_id) == XML_MODULESTORE_TYPE:
         return '/static/' + (course.static_asset_path or getattr(course, 'data_dir', '')) + "/images/course_image.jpg"
     else:
-        loc = course.location._replace(tag='c4x', category='asset', name=course.course_image)
+        loc = course.location.replace(tag='c4x', category='asset', name=course.course_image)
         _path = StaticContent.get_url_path_from_location(loc)
         return _path
 
 
-def find_file(fs, dirs, filename):
+def find_file(filesystem, dirs, filename):
     """
     Looks for a filename in a list of dirs on a filesystem, in the specified order.
 
-    fs: an OSFS filesystem
+    filesystem: an OSFS filesystem
     dirs: a list of path objects
     filename: a string
 
     Returns d / filename if found in dir d, else raises ResourceNotFoundError.
     """
-    for d in dirs:
-        filepath = path(d) / filename
-        if fs.exists(filepath):
+    for directory in dirs:
+        filepath = path(directory) / filename
+        if filesystem.exists(filepath):
             return filepath
     raise ResourceNotFoundError("Could not find {0}".format(filename))
 
@@ -167,7 +167,7 @@ def get_course_about_section(course, section_key):
 
             request = get_request_for_thread()
 
-            loc = course.location._replace(category='about', name=section_key)
+            loc = course.location.replace(category='about', name=section_key)
 
             # Use an empty cache
             field_data_cache = FieldDataCache([], course.id, request.user)
@@ -255,13 +255,13 @@ def get_course_syllabus_section(course, section_key):
 
     if section_key in ['syllabus', 'guest_syllabus']:
         try:
-            fs = course.system.resources_fs
+            filesys = course.system.resources_fs
             # first look for a run-specific version
             dirs = [path("syllabus") / course.url_name, path("syllabus")]
-            filepath = find_file(fs, dirs, section_key + ".html")
-            with fs.open(filepath) as htmlFile:
+            filepath = find_file(filesys, dirs, section_key + ".html")
+            with filesys.open(filepath) as html_file:
                 return replace_static_urls(
-                    htmlFile.read().decode('utf-8'),
+                    html_file.read().decode('utf-8'),
                     getattr(course, 'data_dir', None),
                     course_id=course.location.course_id,
                     static_asset_path=course.static_asset_path,

lms/djangoapps/courseware/tests/test_courses.py

@@ -1,4 +1,7 @@
 # -*- coding: utf-8 -*-
+"""
+Tests for course access
+"""
 import mock
 
 from django.test import TestCase
@@ -44,13 +47,12 @@ class CoursesTest(TestCase):
         self.assertEqual("//{}/".format(CMS_BASE_TEST), get_cms_course_link_by_id("too/too/many/slashes"))
         self.assertEqual("//{}/org/num/course/name".format(CMS_BASE_TEST), get_cms_course_link_by_id('org/num/name'))
 
-
     @mock.patch('xmodule.modulestore.django.get_current_request_hostname', mock.Mock(return_value='preview.localhost'))
-    @override_settings(HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS={'preview\.': 'draft'})
-    def test_default_modulestore_preview_mapping(self):   
+    @override_settings(HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS={r'preview\.': 'draft'})
+    def test_default_modulestore_preview_mapping(self):
         self.assertEqual(get_default_store_name_for_current_request(), 'draft')
 
     @mock.patch('xmodule.modulestore.django.get_current_request_hostname', mock.Mock(return_value='localhost'))
-    @override_settings(HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS={'preview\.': 'draft'})
+    @override_settings(HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS={r'preview\.': 'draft'})
     def test_default_modulestore_published_mapping(self):
         self.assertEqual(get_default_store_name_for_current_request(), 'default')