Allow courses api to return data incrementally

Prior to this commit, the course api (/api/courses/v1/courses/)
performed all the work necessary to return all courses available
to the user, and then only actually returned on page's worth of those
courses.

With this change, the api now does the work incrementally, computing
only the data needed to fetch the courses up to and including the page
being returned. This still increases approximately linearly as
the page number accessed being increases, but should be more cache-friendly.
One side effect of this is that the max_page reported by pagination
will be an overestimate (it will include pages that are removed due
to a users access restrictions).

This change also changes the sort-order of courses being returned by the
course_api. By sorting by course-id, rather than course-number, we
can sort in the database, rather than in Python, and defer loading data
from the end of the list until it is requested.

REVMI-90

openedx/core/djangoapps/content/course_overviews/models.py

@@ -409,7 +409,8 @@ class CourseOverview(TimeStampedModel):
         migrate and test switching to display_name_with_default, which is no
         longer escaped.
         """
-        return block_metadata_utils.display_name_with_default_escaped(self)
+        # pylint: disable=line-too-long
+        return block_metadata_utils.display_name_with_default_escaped(self)  # xss-lint: disable=python-deprecated-display-name
 
     @property
     def dashboard_start_display(self):
@@ -560,7 +561,7 @@ class CourseOverview(TimeStampedModel):
     @classmethod
     def get_all_courses(cls, orgs=None, filter_=None):
         """
-        Returns all CourseOverview objects in the database.
+        Return a queryset containing all CourseOverview objects in the database.
 
         Arguments:
             orgs (list[string]): Optional parameter that allows case-insensitive

openedx/core/lib/api/view_utils.py

@@ -1,6 +1,7 @@
 """
 Utilities related to API views
 """
+from collections import Sequence
 from django.core.exceptions import NON_FIELD_ERRORS, ObjectDoesNotExist, ValidationError
 from django.http import Http404
 from django.utils.translation import ugettext as _
@@ -205,3 +206,100 @@ class RetrievePatchAPIView(RetrieveModelMixin, UpdateModelMixin, GenericAPIView)
                 # PATCH requests where the object does not exist should still
                 # return a 404 response.
                 raise
+
+
+class LazySequence(Sequence):
+    """
+    This class provides an immutable Sequence interface on top of an existing
+    iterable.
+
+    It is immutable, and accepts an estimated length in order to support __len__
+    without exhausting the underlying sequence
+    """
+    def __init__(self, iterable, est_len=None):  # pylint: disable=super-init-not-called
+        self.iterable = iterable
+        self.est_len = est_len
+        self._data = []
+        self._exhausted = False
+
+    def __len__(self):
+        # Return the actual data length if we know it exactly (because
+        # the underlying sequence is exhausted), or it's greater than
+        # the initial estimated length
+        if len(self._data) > self.est_len or self._exhausted:
+            return len(self._data)
+        else:
+            return self.est_len
+
+    def __iter__(self):
+        # Yield all the known data first
+        for item in self._data:
+            yield item
+
+        # Capture and yield data from the underlying iterator
+        # until it is exhausted
+        while True:
+            try:
+                item = next(self.iterable)
+                self._data.append(item)
+                yield item
+            except StopIteration:
+                self._exhausted = True
+                return
+
+    def __getitem__(self, index):
+        if isinstance(index, int):
+            # For a single index, if we haven't already loaded enough
+            # data, we can load data until we have enough, and then
+            # return the value from the loaded data
+            if index < 0:
+                raise IndexError("Negative indexes aren't supported")
+
+            while len(self._data) <= index:
+                try:
+                    self._data.append(next(self.iterable))
+                except StopIteration:
+                    self._exhausted = True
+                    raise IndexError("Underlying sequence exhausted")
+
+            return self._data[index]
+        elif isinstance(index, slice):
+            # For a slice, we can load data until we reach 'stop'.
+            # Once we have data including 'stop', then we can use
+            # the underlying list to actually understand the mechanics
+            # of the slicing operation.
+            if index.start is not None and index.start < 0:
+                raise IndexError("Negative indexes aren't supported")
+            if index.stop is not None and index.stop < 0:
+                raise IndexError("Negative indexes aren't supported")
+
+            if index.step is not None and index.step < 0:
+                largest_value = index.start + 1
+            else:
+                largest_value = index.stop
+
+            if largest_value is not None:
+                while len(self._data) <= largest_value:
+                    try:
+                        self._data.append(next(self.iterable))
+                    except StopIteration:
+                        self._exhausted = True
+                        break
+            else:
+                self._data.extend(self.iterable)
+            return self._data[index]
+        else:
+            raise TypeError("Unsupported index type")
+
+    def __repr__(self):
+        if self._exhausted:
+            return "LazySequence({!r}, {!r})".format(
+                self._data,
+                self.est_len,
+            )
+        else:
+            return "LazySequence(itertools.chain({!r}, {!r}), {!r})".format(
+                self._data,
+                self.iterable,
+                self.est_len,
+            )