Allow courses api to return data incrementally

Prior to this commit, the course api (/api/courses/v1/courses/)
performed all the work necessary to return all courses available
to the user, and then only actually returned on page's worth of those
courses.

With this change, the api now does the work incrementally, computing
only the data needed to fetch the courses up to and including the page
being returned. This still increases approximately linearly as
the page number accessed being increases, but should be more cache-friendly.
One side effect of this is that the max_page reported by pagination
will be an overestimate (it will include pages that are removed due
to a users access restrictions).

This change also changes the sort-order of courses being returned by the
course_api. By sorting by course-id, rather than course-number, we
can sort in the database, rather than in Python, and defer loading data
from the end of the list until it is requested.

REVMI-90

lms/djangoapps/course_api/views.py

@@ -10,6 +10,7 @@ from rest_framework.throttling import UserRateThrottle
 
 from edx_rest_framework_extensions.paginators import NamespacedPageNumberPagination
 from openedx.core.lib.api.view_utils import DeveloperErrorViewMixin, view_auth_classes
+from openedx.core.lib.api.view_utils import LazySequence
 
 from . import USE_RATE_LIMIT_2_FOR_COURSE_LIST_API, USE_RATE_LIMIT_10_FOR_COURSE_LIST_API
 from .api import course_detail, list_courses
@@ -243,7 +244,7 @@ class CourseListView(DeveloperErrorViewMixin, ListAPIView):
 
     def get_queryset(self):
         """
-        Return a list of courses visible to the user.
+        Yield courses visible to the user.
         """
         form = CourseListGetForm(self.request.query_params, initial={'requesting_user': self.request.user})
         if not form.is_valid():
@@ -264,9 +265,12 @@ class CourseListView(DeveloperErrorViewMixin, ListAPIView):
             size=self.results_size_infinity,
         )
 
-        search_courses_ids = {course['data']['id']: True for course in search_courses['results']}
+        search_courses_ids = {course['data']['id'] for course in search_courses['results']}
 
-        return [
-            course for course in db_courses
-            if unicode(course.id) in search_courses_ids
-        ]
+        return LazySequence(
+            (
+                course for course in db_courses
+                if unicode(course.id) in search_courses_ids
+            ),
+            est_len=len(db_courses)
+        )