Allow courses api to return data incrementally

Prior to this commit, the course api (/api/courses/v1/courses/)
performed all the work necessary to return all courses available
to the user, and then only actually returned on page's worth of those
courses.

With this change, the api now does the work incrementally, computing
only the data needed to fetch the courses up to and including the page
being returned. This still increases approximately linearly as
the page number accessed being increases, but should be more cache-friendly.
One side effect of this is that the max_page reported by pagination
will be an overestimate (it will include pages that are removed due
to a users access restrictions).

This change also changes the sort-order of courses being returned by the
course_api. By sorting by course-id, rather than course-number, we
can sort in the database, rather than in Python, and defer loading data
from the end of the list until it is requested.

REVMI-90

lms/djangoapps/courseware/courses.py

@@ -35,6 +35,7 @@ from lms.djangoapps.courseware.exceptions import CourseAccessRedirect
 from opaque_keys.edx.keys import UsageKey
 from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
 from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers
+from openedx.core.lib.api.view_utils import LazySequence
 from path import Path as path
 from six import text_type
 from static_replace import replace_static_urls
@@ -451,8 +452,7 @@ def get_course_syllabus_section(course, section_key):
 
 def get_courses(user, org=None, filter_=None):
     """
-    Returns a list of courses available, sorted by course.number and optionally
-    filtered by org code (case-insensitive).
+    Return a LazySequence of courses available, optionally filtered by org code (case-insensitive).
     """
     courses = branding.get_visible_courses(org=org, filter_=filter_)
 
@@ -461,9 +461,10 @@ def get_courses(user, org=None, filter_=None):
         settings.COURSE_CATALOG_VISIBILITY_PERMISSION
     )
 
-    courses = [c for c in courses if has_access(user, permission_name, c)]
-
-    return courses
+    return LazySequence(
+        (c for c in courses if has_access(user, permission_name, c)),
+        est_len=courses.count()
+    )
 
 
 def get_permission_for_course_about():

lms/djangoapps/courseware/tests/test_courses.py

@@ -141,7 +141,7 @@ class CoursesTest(ModuleStoreTestCase):
 
             # Request filtering for an org distinct from the designated org.
             no_courses = get_courses(user, org=primary)
-            self.assertEqual(no_courses, [])
+            self.assertEqual(list(no_courses), [])
 
             # Request filtering for an org matching the designated org.
             site_courses = get_courses(user, org=alternate)