feat: Implement Sidebar Navigation [FC-0056] (#34457)

This commit addresses to the need to add an API for Sidebar Navigation
that returns the course structure with sections, subsections, and
**units**, according to user access rules. (Previous outline requests
only went down to subsections.)

To improve the performance of the API, we cache the course structure
for a user, which makes it much easier to calculate the block structure
for the user at each request. However, there may be cases when this
caching can lead to an overflow of the cache storage in high-loaded LMS
with large courses, so the corresponding flag
"courseware.disable_navigation_sidebar_blocks_caching" was added so
that this caching can be disabled.
This commit is contained in:
Ivan Niedielnitsev
2024-04-26 19:04:42 +03:00
committed by GitHub
parent 9630d4080a
commit 30836729b7
5 changed files with 504 additions and 2 deletions

View File

@@ -19,7 +19,8 @@ class CourseBlockSerializer(serializers.Serializer):
def get_blocks(self, block): # pylint: disable=missing-function-docstring
block_key = block['id']
block_type = block['type']
children = block.get('children', []) if block_type != 'sequential' else [] # Don't descend past sequential
last_parent_block_type = 'vertical' if self.context.get('include_vertical') else 'sequential'
children = block.get('children', []) if block_type != last_parent_block_type else []
description = block.get('format')
display_name = block['display_name']
enable_links = self.context.get('enable_links')
@@ -35,10 +36,16 @@ class CourseBlockSerializer(serializers.Serializer):
if graded and scored:
icon = 'fa-pencil-square-o'
if block_type == 'vertical':
icon = self.get_vertical_icon_class(block)
if 'special_exam_info' in block:
description = block['special_exam_info'].get('short_description')
icon = block['special_exam_info'].get('suggested_icon', 'fa-pencil-square-o')
if self.context.get('enable_prerequisite_block_type', False) and block.get('accessible') is False:
block_type = 'lock'
serialized = {
block_key: {
'children': [child['id'] for child in children],
@@ -57,10 +64,29 @@ class CourseBlockSerializer(serializers.Serializer):
'hide_from_toc': block.get('hide_from_toc'),
},
}
if 'special_exam_info' in self.context.get('extra_fields', []) and block.get('special_exam_info'):
serialized[block_key]['special_exam_info'] = block.get('special_exam_info').get('short_description')
for child in children:
serialized.update(self.get_blocks(child))
return serialized
@staticmethod
def get_vertical_icon_class(block):
"""
Get the icon class for a vertical block based priority of child blocks types.
Currently, the priority for the icon is as follows:
problem
video
"""
children = block.get('children', [])
child_classes = {child.get('type') for child in children}
if 'problem' in child_classes:
return 'problem'
if 'video' in child_classes:
return 'video'
return 'other'
class CourseGoalsSerializer(serializers.Serializer):
"""

View File

@@ -447,3 +447,242 @@ class OutlineTabTestViews(BaseCourseHomeTests):
self.update_course_and_overview()
CourseEnrollment.enroll(UserFactory(), self.course.id) # grr, some rando took our spot!
self.assert_can_enroll(False)
@ddt.ddt
class SidebarBlocksTestViews(BaseCourseHomeTests):
"""
Tests for the Course Sidebar Blocks API
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.chapter = ''
self.sequential = ''
self.vertical = ''
self.ungraded_sequential = ''
self.ungraded_vertical = ''
self.url = ''
def setUp(self):
super().setUp()
self.url = reverse('course-home:course-navigation', args=[self.course.id])
def update_course_and_overview(self):
"""
Update the course and course overview records.
"""
self.update_course(self.course, self.user.id)
CourseOverview.load_from_module_store(self.course.id)
def add_blocks_to_course(self):
"""
Add test blocks to the self course.
"""
with self.store.bulk_operations(self.course.id):
self.chapter = BlockFactory.create(category='chapter', parent_location=self.course.location)
self.sequential = BlockFactory.create(
display_name='Test',
category='sequential',
graded=True,
has_score=True,
parent_location=self.chapter.location
)
self.vertical = BlockFactory.create(
category='problem',
graded=True,
has_score=True,
parent_location=self.sequential.location
)
self.ungraded_sequential = BlockFactory.create(
display_name='Ungraded',
category='sequential',
parent_location=self.chapter.location
)
self.ungraded_vertical = BlockFactory.create(
category='problem',
parent_location=self.ungraded_sequential.location
)
update_outline_from_modulestore(self.course.id)
@ddt.data(CourseMode.AUDIT, CourseMode.VERIFIED)
def test_get_authenticated_enrolled_user(self, enrollment_mode):
"""
Test that the API returns the correct data for an authenticated, enrolled user.
"""
self.add_blocks_to_course()
CourseEnrollment.enroll(self.user, self.course.id, enrollment_mode)
response = self.client.get(self.url)
assert response.status_code == 200
chapter_data = response.data['blocks'][str(self.chapter.location)]
assert str(self.sequential.location) in chapter_data['children']
sequential_data = response.data['blocks'][str(self.sequential.location)]
assert str(self.vertical.location) in sequential_data['children']
vertical_data = response.data['blocks'][str(self.vertical.location)]
assert vertical_data['children'] == []
@ddt.data(True, False)
def test_get_authenticated_user_not_enrolled(self, has_previously_enrolled):
"""
Test that the API returns an empty response for an authenticated user who is not enrolled in the course.
"""
if has_previously_enrolled:
CourseEnrollment.enroll(self.user, self.course.id)
CourseEnrollment.unenroll(self.user, self.course.id)
response = self.client.get(self.url)
assert response.status_code == 200
assert response.data == {}
def test_get_unauthenticated_user(self):
"""
Test that the API returns an empty response for an unauthenticated user.
"""
self.client.logout()
response = self.client.get(self.url)
assert response.status_code == 200
assert response.data.get('blocks') is None
def test_course_staff_can_see_non_user_specific_content_in_masquerade(self):
"""
Test that course staff can see the outline and other non-user-specific content when masquerading as a learner
"""
instructor = UserFactory(username='instructor', email='instructor@example.com', password='foo', is_staff=False)
CourseInstructorRole(self.course.id).add_users(instructor)
self.client.login(username=instructor, password='foo')
self.update_masquerade(role='student')
response = self.client.get(self.url)
assert response.data['blocks'] is not None
def test_get_unknown_course(self):
"""
Test that the API returns a 404 when the course is not found.
"""
url = reverse('course-home:course-navigation', args=['course-v1:unknown+course+2T2020'])
response = self.client.get(url)
assert response.status_code == 404
@patch.dict('django.conf.settings.FEATURES', {'ENABLE_SPECIAL_EXAMS': True})
@patch('lms.djangoapps.course_api.blocks.transformers.milestones.get_attempt_status_summary')
def test_proctored_exam(self, mock_summary):
"""
Test that the API returns the correct data for a proctored exam.
"""
course = CourseFactory.create(
org='edX',
course='900',
run='test_run',
enable_proctored_exams=True,
proctoring_provider=settings.PROCTORING_BACKENDS['DEFAULT'],
)
chapter = BlockFactory.create(parent=course, category='chapter', display_name='Test Section')
sequence = BlockFactory.create(
parent=chapter,
category='sequential',
display_name='Test Proctored Exam',
graded=True,
is_time_limited=True,
default_time_limit_minutes=10,
is_practice_exam=True,
due=datetime.now(),
exam_review_rules='allow_use_of_paper',
hide_after_due=False,
is_onboarding_exam=False,
)
sequence.is_proctored_exam = True
update_outline_from_modulestore(course.id)
CourseEnrollment.enroll(self.user, course.id)
mock_summary.return_value = {
'short_description': 'My Exam',
'suggested_icon': 'fa-foo-bar',
}
url = reverse('course-home:course-navigation', args=[course.id])
response = self.client.get(url)
assert response.status_code == 200
exam_data = response.data['blocks'][str(sequence.location)]
assert not exam_data['complete']
assert exam_data['display_name'] == 'Test Proctored Exam'
assert exam_data['special_exam_info'] == 'My Exam'
assert exam_data['due'] is not None
def test_assignment(self):
"""
Test that the API returns the correct data for an assignment.
"""
self.add_blocks_to_course()
CourseEnrollment.enroll(self.user, self.course.id)
response = self.client.get(self.url)
assert response.status_code == 200
exam_data = response.data['blocks'][str(self.sequential.location)]
assert exam_data['display_name'] == 'Test (1 Question)'
assert exam_data['icon'] == 'fa-pencil-square-o'
assert str(self.vertical.location) in exam_data['children']
ungraded_data = response.data['blocks'][str(self.ungraded_sequential.location)]
assert ungraded_data['display_name'] == 'Ungraded'
assert ungraded_data['icon'] is None
assert str(self.ungraded_vertical.location) in ungraded_data['children']
@override_waffle_flag(COURSE_ENABLE_UNENROLLED_ACCESS_FLAG, active=True)
@ddt.data(*itertools.product(
[True, False], [True, False], [None, COURSE_VISIBILITY_PUBLIC, COURSE_VISIBILITY_PUBLIC_OUTLINE]
))
@ddt.unpack
def test_visibility(self, is_enrolled, is_staff, course_visibility):
"""
Test that the API returns the correct data based on the user's enrollment status and the course's visibility.
"""
if is_enrolled:
CourseEnrollment.enroll(self.user, self.course.id)
if is_staff:
self.user.is_staff = True
self.user.save()
if course_visibility:
self.course.course_visibility = course_visibility
self.update_course_and_overview()
show_enrolled = is_enrolled or is_staff
is_public = course_visibility == COURSE_VISIBILITY_PUBLIC
is_public_outline = course_visibility == COURSE_VISIBILITY_PUBLIC_OUTLINE
data = self.client.get(self.url).data
if not (show_enrolled or is_public or is_public_outline):
assert data == {}
else:
assert (data['blocks'] is not None) == (show_enrolled or is_public or is_public_outline)
def test_hide_learning_sequences(self):
"""
Check that Learning Sequences filters out sequences.
"""
CourseEnrollment.enroll(self.user, self.course.id, CourseMode.VERIFIED)
response = self.client.get(self.url)
assert response.status_code == 200
blocks = response.data['blocks']
seq_block_id = next(block_id for block_id, block in blocks.items() if block['type'] in ('sequential', 'lock'))
# With a course outline loaded, the same sequence is removed.
new_learning_seq_outline = CourseOutlineData(
course_key=self.course.id,
title='Test Course Outline!',
published_at=datetime(2021, 6, 14, tzinfo=timezone.utc),
published_version='5ebece4b69dd593d82fe2022',
entrance_exam_id=None,
days_early_for_beta=None,
sections=[],
self_paced=False,
course_visibility=CourseVisibility.PRIVATE
)
replace_course_outline(new_learning_seq_outline)
blocks = self.client.get(self.url).data['blocks']
assert seq_block_id not in blocks

View File

@@ -2,10 +2,13 @@
Outline Tab Views
"""
from datetime import datetime, timezone
from functools import cached_property
from completion.exceptions import UnavailableCompletionData # lint-amnesty, pylint: disable=wrong-import-order
from completion.models import BlockCompletion
from completion.utilities import get_key_to_last_completed_block # lint-amnesty, pylint: disable=wrong-import-order
from django.conf import settings # lint-amnesty, pylint: disable=wrong-import-order
from django.core.cache import cache
from django.shortcuts import get_object_or_404 # lint-amnesty, pylint: disable=wrong-import-order
from django.urls import reverse # lint-amnesty, pylint: disable=wrong-import-order
from django.utils.translation import gettext as _ # lint-amnesty, pylint: disable=wrong-import-order
@@ -28,6 +31,7 @@ from lms.djangoapps.course_goals.api import (
)
from lms.djangoapps.course_goals.models import CourseGoal
from lms.djangoapps.course_home_api.outline.serializers import (
CourseBlockSerializer,
OutlineTabSerializer,
)
from lms.djangoapps.course_home_api.utils import get_course_or_403
@@ -36,11 +40,13 @@ from lms.djangoapps.courseware.context_processor import user_timezone_locale_pre
from lms.djangoapps.courseware.courses import get_course_date_blocks, get_course_info_section
from lms.djangoapps.courseware.date_summary import TodaysDate
from lms.djangoapps.courseware.masquerade import is_masquerading, setup_masquerade
from lms.djangoapps.courseware.toggles import courseware_disable_navigation_sidebar_blocks_caching
from lms.djangoapps.courseware.views.views import get_cert_data
from lms.djangoapps.grades.course_grade_factory import CourseGradeFactory
from lms.djangoapps.utils import OptimizelyClient
from openedx.core.djangoapps.content.learning_sequences.api import get_user_course_outline
from openedx.core.djangoapps.content.course_overviews.api import get_course_overview_or_404
from openedx.core.djangoapps.course_groups.cohorts import get_cohort
from openedx.core.lib.api.authentication import BearerAuthenticationAllowInactiveUser
from openedx.features.course_duration_limits.access import get_access_expiration_data
from openedx.features.course_experience import COURSE_ENABLE_UNENROLLED_ACCESS_FLAG, ENABLE_COURSE_GOALS
@@ -52,6 +58,8 @@ from openedx.features.course_experience.course_updates import (
from openedx.features.course_experience.url_helpers import get_learning_mfe_home_url
from openedx.features.course_experience.utils import get_course_outline_block_tree, get_start_block
from openedx.features.discounts.utils import generate_offer_data
from xblock.core import XBlock
from xblock.completable import XBlockCompletionMode
from xmodule.course_block import COURSE_VISIBILITY_PUBLIC, COURSE_VISIBILITY_PUBLIC_OUTLINE # lint-amnesty, pylint: disable=wrong-import-order
@@ -375,6 +383,204 @@ class OutlineTabView(RetrieveAPIView):
return expose_header('Date', response)
class CourseNavigationBlocksView(RetrieveAPIView):
"""
**Use Cases**
Request details for the sidebar navigation of the course.
**Example Requests**
GET api/course_home/v1/navigation/{course_key}
**Response Values**
For a good 200 response, the response will include:
blocks: List of serialized Course Block objects. Each serialization has the following fields:
id: (str) The usage ID of the block.
type: (str) The type of block. Possible values the names of any
XBlock type in the system, including custom blocks. Examples are
course, chapter, sequential, vertical, html, problem, video, and
discussion.
display_name: (str) The display name of the block.
lms_web_url: (str) The URL to the navigational container of the
xBlock on the web LMS.
children: (list) If the block has child blocks, a list of IDs of
the child blocks.
resume_block: (bool) Whether the block is the resume block
has_scheduled_content: (bool) Whether the block has more content scheduled for the future
**Returns**
* 200 on success.
* 403 if the user does not currently have access to the course and should be redirected.
* 404 if the course is not available or cannot be seen.
"""
authentication_classes = (
JwtAuthentication,
BearerAuthenticationAllowInactiveUser,
SessionAuthenticationAllowInactiveUser,
)
serializer_class = CourseBlockSerializer
COURSE_BLOCKS_CACHE_KEY_TEMPLATE = (
'course_sidebar_blocks_{course_key_string}_{course_version}_{user_id}_{user_cohort_id}'
'_{enrollment_mode}_{allow_public}_{allow_public_outline}_{is_masquerading}'
)
COURSE_BLOCKS_CACHE_TIMEOUT = 60 * 60 # 1 hour
def get(self, request, *args, **kwargs):
"""
Get the visible course blocks (from course to vertical types) for the given course.
"""
course_key_string = kwargs.get('course_key_string')
course_key = CourseKey.from_string(course_key_string)
course = get_course_or_403(request.user, 'load', course_key, check_if_enrolled=False)
staff_access = has_access(request.user, 'staff', course_key)
masquerade_object, request.user = setup_masquerade(
request,
course_key,
staff_access=staff_access,
reset_masquerade_data=True,
)
user_is_masquerading = is_masquerading(request.user, course_key, course_masquerade=masquerade_object)
allow_anonymous = COURSE_ENABLE_UNENROLLED_ACCESS_FLAG.is_enabled(course_key)
allow_public = allow_anonymous and course.course_visibility == COURSE_VISIBILITY_PUBLIC
allow_public_outline = allow_anonymous and course.course_visibility == COURSE_VISIBILITY_PUBLIC_OUTLINE
enrollment = CourseEnrollment.get_enrollment(request.user, course_key)
try:
user_cohort = get_cohort(request.user, course_key, use_cached=True)
except ValueError:
user_cohort = None
cache_key = self.COURSE_BLOCKS_CACHE_KEY_TEMPLATE.format(
course_key_string=course_key_string,
course_version=str(course.course_version),
user_id=request.user.id,
enrollment_mode=getattr(enrollment, 'mode', ''),
user_cohort_id=getattr(user_cohort, 'id', ''),
allow_public=allow_public,
allow_public_outline=allow_public_outline,
is_masquerading=user_is_masquerading,
)
if navigation_sidebar_caching_is_disabled := courseware_disable_navigation_sidebar_blocks_caching():
cached = False
course_blocks = None
else:
course_blocks = cache.get(cache_key)
cached = course_blocks is not None
if not course_blocks:
if getattr(enrollment, 'is_active', False) or bool(staff_access):
course_blocks = get_course_outline_block_tree(request, course_key_string, request.user)
elif allow_public_outline or allow_public or user_is_masquerading:
course_blocks = get_course_outline_block_tree(request, course_key_string, None)
if not navigation_sidebar_caching_is_disabled:
cache.set(cache_key, course_blocks, self.COURSE_BLOCKS_CACHE_TIMEOUT)
course_blocks = self.filter_inaccessible_blocks(course_blocks, course_key)
if cached:
# Note: The course_blocks received from get_course_outline_block_tree already has completion data,
# but since the course_blocks can be cached, and this status can change quite often,
# we need to update it every time if the data has not been cached.
course_blocks = self.mark_complete_recursive(course_blocks)
context = self.get_serializer_context()
context.update({
'include_vertical': True,
'extra_fields': ['special_exam_info'],
'enable_prerequisite_block_type': True,
})
serializer = self.get_serializer_class()(course_blocks, context=context)
return Response(serializer.data)
def filter_inaccessible_blocks(self, course_blocks, course_key):
"""
Filter out sections and subsections that are not accessible to the current user.
"""
if course_blocks:
user_course_outline = get_user_course_outline(course_key, self.request.user, datetime.now(tz=timezone.utc))
course_sections = course_blocks.get('children', [])
course_blocks['children'] = self.get_accessible_sections(user_course_outline, course_sections)
for section_data in course_sections:
section_data['children'] = self.get_accessible_sequences(
user_course_outline,
section_data.get('children', [])
)
accessible_sequence_ids = {str(usage_key) for usage_key in user_course_outline.accessible_sequences}
for sequence_data in section_data['children']:
sequence_data['accessible'] = sequence_data['id'] in accessible_sequence_ids
return course_blocks
def mark_complete_recursive(self, block):
"""
Mark blocks as complete or not complete based on the completions_dict.
"""
if 'children' in block:
block['children'] = [self.mark_complete_recursive(child) for child in block['children']]
block['complete'] = all(
child['complete'] for child in block['children'] if child['type'] in self.completable_block_types
)
else:
block['complete'] = self.completions_dict.get(block['id'], False)
return block
@staticmethod
def get_accessible_sections(user_course_outline, course_sections):
"""
Filter out sections that are not accessible to the user.
"""
available_section_ids = set(map(lambda section: str(section.usage_key), user_course_outline.sections))
return [
section_data for section_data in course_sections
if section_data['id'] in available_section_ids
]
@staticmethod
def get_accessible_sequences(user_course_outline, course_sequences):
"""
Filter out sequences that are not accessible to the user.
"""
available_sequence_ids = set(map(str, user_course_outline.sequences))
return [
seq_data for seq_data in course_sequences
if seq_data['id'] in available_sequence_ids or seq_data['type'] != 'sequential'
]
@cached_property
def completions_dict(self):
"""
Return a dictionary of block completions for the current user.
Dictionary keys are block keys and values are int values
representing the completion status of the block.
"""
course_key_string = self.kwargs.get('course_key_string')
course_key = CourseKey.from_string(course_key_string)
completions = BlockCompletion.objects.filter(user=self.request.user, context_key=course_key).values_list(
'block_key',
'completion',
)
return {
str(block_key): completion
for block_key, completion in completions
}
@cached_property
def completable_block_types(self):
"""
Return a set of block types that are completable.
"""
return {
block_type for (block_type, block_cls) in XBlock.load_classes()
if XBlockCompletionMode.get_mode(block_cls) == XBlockCompletionMode.COMPLETABLE
}
@api_view(['POST'])
@permission_classes((IsAuthenticated,))
def dismiss_welcome_message(request): # pylint: disable=missing-function-docstring

View File

@@ -9,7 +9,11 @@ from django.urls import re_path
from lms.djangoapps.course_home_api.course_metadata.views import CourseHomeMetadataView
from lms.djangoapps.course_home_api.dates.views import DatesTabView
from lms.djangoapps.course_home_api.outline.views import (
OutlineTabView, dismiss_welcome_message, save_course_goal, unsubscribe_from_course_goal_by_token,
CourseNavigationBlocksView,
OutlineTabView,
dismiss_welcome_message,
save_course_goal,
unsubscribe_from_course_goal_by_token,
)
from lms.djangoapps.course_home_api.progress.views import ProgressTabView
@@ -44,6 +48,11 @@ urlpatterns += [
OutlineTabView.as_view(),
name='outline-tab'
),
re_path(
fr'navigation/{settings.COURSE_KEY_PATTERN}',
CourseNavigationBlocksView.as_view(),
name='course-navigation'
),
re_path(
r'dismiss_welcome_message',
dismiss_welcome_message,

View File

@@ -68,6 +68,21 @@ COURSEWARE_MICROFRONTEND_SEARCH_ENABLED = CourseWaffleFlag(
f'{WAFFLE_FLAG_NAMESPACE}.mfe_courseware_search', __name__
)
# .. toggle_name: courseware.disable_navigation_sidebar_blocks_caching
# .. toggle_implementation: WaffleFlag
# .. toggle_default: False
# .. toggle_description: Disable caching of navigation sidebar blocks on Learning MFE.
# It can be used when caching the structure of large courses for a large number of users
# at the same time can overload the cache storage (memcache or redis).
# .. toggle_use_cases: temporary
# .. toggle_creation_date: 2024-03-21
# .. toggle_target_removal_date: None
# .. toggle_tickets: FC-0056
# .. toggle_warning: None.
COURSEWARE_MICROFRONTEND_NAVIGATION_SIDEBAR_BLOCKS_DISABLE_CACHING = CourseWaffleFlag(
f'{WAFFLE_FLAG_NAMESPACE}.disable_navigation_sidebar_blocks_caching', __name__
)
# .. toggle_name: courseware.mfe_progress_milestones_streak_discount_enabled
# .. toggle_implementation: CourseWaffleFlag
# .. toggle_default: False
@@ -170,3 +185,10 @@ def courseware_mfe_search_is_enabled(course_key=None):
Return whether the courseware.mfe_courseware_search flag is on.
"""
return COURSEWARE_MICROFRONTEND_SEARCH_ENABLED.is_enabled(course_key)
def courseware_disable_navigation_sidebar_blocks_caching(course_key=None):
"""
Return whether the courseware.disable_navigation_sidebar_blocks_caching flag is on.
"""
return COURSEWARE_MICROFRONTEND_NAVIGATION_SIDEBAR_BLOCKS_DISABLE_CACHING.is_enabled(course_key)