From fd5fc29dd6fd211bf9c85d68f860fc7ca990f12d Mon Sep 17 00:00:00 2001 From: Kyrylo Kholodenko Date: Tue, 8 Apr 2025 11:02:35 +0300 Subject: [PATCH 01/54] feat: api for shifting all relative past due dates --- .../api/v1/tests/test_utils.py | 84 +++++++++++++++ .../api/v1/tests/test_views.py | 100 ++++++++++-------- .../features/course_experience/api/v1/urls.py | 11 +- .../course_experience/api/v1/utils.py | 51 +++++++++ .../course_experience/api/v1/views.py | 81 ++++++++------ 5 files changed, 250 insertions(+), 77 deletions(-) create mode 100644 openedx/features/course_experience/api/v1/tests/test_utils.py create mode 100644 openedx/features/course_experience/api/v1/utils.py diff --git a/openedx/features/course_experience/api/v1/tests/test_utils.py b/openedx/features/course_experience/api/v1/tests/test_utils.py new file mode 100644 index 0000000000..10b5882918 --- /dev/null +++ b/openedx/features/course_experience/api/v1/tests/test_utils.py @@ -0,0 +1,84 @@ +""" +Tests utils of course expirience feature. +""" +import datetime + +from django.urls import reverse +from django.utils import timezone +from rest_framework.test import APIRequestFactory + +from common.djangoapps.course_modes.models import CourseMode +from common.djangoapps.student.models import CourseEnrollment +from common.djangoapps.util.testing import EventTestMixin +from lms.djangoapps.course_home_api.tests.utils import BaseCourseHomeTests +from lms.djangoapps.courseware.tests.helpers import MasqueradeMixin +from openedx.core.djangoapps.schedules.models import Schedule +from openedx.features.course_experience.api.v1.utils import reset_deadlines_for_course +from xmodule.modulestore.tests.factories import CourseFactory + + +class TestResetDeadlinesForCourse(EventTestMixin, BaseCourseHomeTests, MasqueradeMixin): + """ + Tests for reset deadlines endpoint. + """ + def setUp(self): # pylint: disable=arguments-differ + super().setUp("openedx.features.course_experience.api.v1.utils.tracker") + self.course = CourseFactory.create(self_paced=True, start=timezone.now() - datetime.timedelta(days=1000)) + + def test_reset_deadlines_for_course(self): + enrollment = CourseEnrollment.enroll(self.user, self.course.id, CourseMode.VERIFIED) + enrollment.schedule.start_date = timezone.now() - datetime.timedelta(days=100) + enrollment.schedule.save() + + request = APIRequestFactory().post( + reverse("course-experience-reset-course-deadlines"), {"course_key": self.course.id} + ) + request.user = self.user + + reset_deadlines_for_course(request, self.course.id, {}) + + assert enrollment.schedule.start_date < Schedule.objects.get(id=enrollment.schedule.id).start_date + self.assert_event_emitted( + "edx.ui.lms.reset_deadlines.clicked", + courserun_key=str(self.course.id), + is_masquerading=False, + is_staff=False, + org_key=self.course.org, + user_id=self.user.id, + ) + + def test_reset_deadlines_with_masquerade(self): + """Staff users should be able to masquerade as a learner and reset the learner's schedule""" + student_username = self.user.username + student_user_id = self.user.id + student_enrollment = CourseEnrollment.enroll(self.user, self.course.id) + student_enrollment.schedule.start_date = timezone.now() - datetime.timedelta(days=100) + student_enrollment.schedule.save() + + staff_enrollment = CourseEnrollment.enroll(self.staff_user, self.course.id) + staff_enrollment.schedule.start_date = timezone.now() - datetime.timedelta(days=30) + staff_enrollment.schedule.save() + + self.switch_to_staff() + self.update_masquerade(course=self.course, username=student_username) + + request = APIRequestFactory().post( + reverse("course-experience-reset-course-deadlines"), {"course_key": self.course.id} + ) + request.user = self.staff_user + request.session = self.client.session + + reset_deadlines_for_course(request, self.course.id, {}) + + updated_schedule = Schedule.objects.get(id=student_enrollment.schedule.id) + assert updated_schedule.start_date.date() == datetime.datetime.today().date() + updated_staff_schedule = Schedule.objects.get(id=staff_enrollment.schedule.id) + assert updated_staff_schedule.start_date == staff_enrollment.schedule.start_date + self.assert_event_emitted( + "edx.ui.lms.reset_deadlines.clicked", + courserun_key=str(self.course.id), + is_masquerading=True, + is_staff=False, + org_key=self.course.org, + user_id=student_user_id, + ) diff --git a/openedx/features/course_experience/api/v1/tests/test_views.py b/openedx/features/course_experience/api/v1/tests/test_views.py index 8cef39053b..d7e8f6cafc 100644 --- a/openedx/features/course_experience/api/v1/tests/test_views.py +++ b/openedx/features/course_experience/api/v1/tests/test_views.py @@ -1,7 +1,9 @@ """ Tests for reset deadlines endpoint. """ + import datetime +from unittest import mock import ddt from django.urls import reverse @@ -10,7 +12,6 @@ from edx_toggles.toggles.testutils import override_waffle_flag from common.djangoapps.course_modes.models import CourseMode from common.djangoapps.student.models import CourseEnrollment -from common.djangoapps.util.testing import EventTestMixin from lms.djangoapps.course_home_api.tests.utils import BaseCourseHomeTests from lms.djangoapps.courseware.tests.helpers import MasqueradeMixin from openedx.core.djangoapps.schedules.models import Schedule @@ -19,14 +20,12 @@ from xmodule.modulestore.tests.factories import CourseFactory @ddt.ddt -class ResetCourseDeadlinesViewTests(EventTestMixin, BaseCourseHomeTests, MasqueradeMixin): +class ResetCourseDeadlinesViewTests(BaseCourseHomeTests, MasqueradeMixin): """ Tests for reset deadlines endpoint. """ def setUp(self): # pylint: disable=arguments-differ - # Need to supply tracker name for the EventTestMixin. Also, EventTestMixin needs to come - # first in class inheritance so the setUp call here appropriately works - super().setUp('openedx.features.course_experience.api.v1.views.tracker') + super().setUp() self.course = CourseFactory.create(self_paced=True, start=timezone.now() - datetime.timedelta(days=1000)) def test_reset_deadlines(self): @@ -37,20 +36,11 @@ class ResetCourseDeadlinesViewTests(EventTestMixin, BaseCourseHomeTests, Masquer response = self.client.post(reverse('course-experience-reset-course-deadlines'), {'course': self.course.id}) assert response.status_code == 400 assert enrollment.schedule == Schedule.objects.get(id=enrollment.schedule.id) - self.assert_no_events_were_emitted() # Test correct post body response = self.client.post(reverse('course-experience-reset-course-deadlines'), {'course_key': self.course.id}) assert response.status_code == 200 assert enrollment.schedule.start_date < Schedule.objects.get(id=enrollment.schedule.id).start_date - self.assert_event_emitted( - 'edx.ui.lms.reset_deadlines.clicked', - courserun_key=str(self.course.id), - is_masquerading=False, - is_staff=False, - org_key=self.course.org, - user_id=self.user.id, - ) @override_waffle_flag(RELATIVE_DATES_FLAG, active=True) @override_waffle_flag(RELATIVE_DATES_DISABLE_RESET_FLAG, active=True) @@ -62,36 +52,6 @@ class ResetCourseDeadlinesViewTests(EventTestMixin, BaseCourseHomeTests, Masquer response = self.client.post(reverse('course-experience-reset-course-deadlines'), {'course_key': self.course.id}) assert response.status_code == 200 assert enrollment.schedule == Schedule.objects.get(id=enrollment.schedule.id) - self.assert_no_events_were_emitted() - - def test_reset_deadlines_with_masquerade(self): - """ Staff users should be able to masquerade as a learner and reset the learner's schedule """ - student_username = self.user.username - student_user_id = self.user.id - student_enrollment = CourseEnrollment.enroll(self.user, self.course.id) - student_enrollment.schedule.start_date = timezone.now() - datetime.timedelta(days=100) - student_enrollment.schedule.save() - - staff_enrollment = CourseEnrollment.enroll(self.staff_user, self.course.id) - staff_enrollment.schedule.start_date = timezone.now() - datetime.timedelta(days=30) - staff_enrollment.schedule.save() - - self.switch_to_staff() - self.update_masquerade(course=self.course, username=student_username) - - self.client.post(reverse('course-experience-reset-course-deadlines'), {'course_key': self.course.id}) - updated_schedule = Schedule.objects.get(id=student_enrollment.schedule.id) - assert updated_schedule.start_date.date() == datetime.datetime.today().date() - updated_staff_schedule = Schedule.objects.get(id=staff_enrollment.schedule.id) - assert updated_staff_schedule.start_date == staff_enrollment.schedule.start_date - self.assert_event_emitted( - 'edx.ui.lms.reset_deadlines.clicked', - courserun_key=str(self.course.id), - is_masquerading=True, - is_staff=False, - org_key=self.course.org, - user_id=student_user_id, - ) def test_post_unauthenticated_user(self): self.client.logout() @@ -115,3 +75,55 @@ class ResetCourseDeadlinesViewTests(EventTestMixin, BaseCourseHomeTests, Masquer self.client.logout() response = self.client.get(reverse('course-experience-course-deadlines-mobile', args=[self.course.id])) assert response.status_code == 401 + + +class ResetAllRelativeCourseDeadlinesViewTests(BaseCourseHomeTests, MasqueradeMixin): + """ + Tests for reset all relative deadlines endpoint. + """ + + def setUp(self): # pylint: disable=arguments-differ + super().setUp() + self.course = CourseFactory.create(self_paced=True, start=timezone.now() - datetime.timedelta(days=1000)) + self.enrollment = CourseEnrollment.enroll(self.user, self.course.id, CourseMode.VERIFIED) + self.enrollment.schedule.start_date = timezone.now() - datetime.timedelta(days=100) + self.enrollment.schedule.save() + + def test_reset_all_relative_course_deadlines(self): + """ + Test reset all relative course deadlines endpoint + """ + response = self.client.post( + reverse("course-experience-reset-all-relative-course-deadlines"), + {}, + ) + assert response.status_code == 200 + assert self.enrollment.schedule.start_date < Schedule.objects.get(id=self.enrollment.schedule.id).start_date + assert str(self.course.id) in response.data.get("success_course_keys") + + def test_reset_all_relative_course_deadlines_failure(self): + """ + Raise exception on reset_deadlines_for_course and assert if failure course id is returned + """ + with mock.patch( + "openedx.features.course_experience.api.v1.views.reset_deadlines_for_course", + side_effect=Exception("Test Exception"), + ): + response = self.client.post( + reverse("course-experience-reset-all-relative-course-deadlines"), + {}, + ) + + assert response.status_code == 200 + assert str(self.course.id) in response.data.get("failed_course_keys") + + def test_post_unauthenticated_user(self): + """ + Test reset all relative course deadlines endpoint for unauthenticated user + """ + self.client.logout() + response = self.client.post( + reverse("course-experience-reset-all-relative-course-deadlines"), + {}, + ) + assert response.status_code == 401 diff --git a/openedx/features/course_experience/api/v1/urls.py b/openedx/features/course_experience/api/v1/urls.py index 9a2c7106cd..7e5a0936c2 100644 --- a/openedx/features/course_experience/api/v1/urls.py +++ b/openedx/features/course_experience/api/v1/urls.py @@ -6,7 +6,11 @@ Contains URLs for the Course Experience API from django.conf import settings from django.urls import re_path -from openedx.features.course_experience.api.v1.views import reset_course_deadlines, CourseDeadlinesMobileView +from openedx.features.course_experience.api.v1.views import ( + reset_course_deadlines, + reset_all_relative_course_deadlines, + CourseDeadlinesMobileView, +) urlpatterns = [] @@ -17,6 +21,11 @@ urlpatterns += [ reset_course_deadlines, name='course-experience-reset-course-deadlines' ), + re_path( + r'v1/reset_all_relative_course_deadlines/', + reset_all_relative_course_deadlines, + name='course-experience-reset-all-relative-course-deadlines', + ) ] # URL for retrieving course deadlines info diff --git a/openedx/features/course_experience/api/v1/utils.py b/openedx/features/course_experience/api/v1/utils.py new file mode 100644 index 0000000000..aabfadb540 --- /dev/null +++ b/openedx/features/course_experience/api/v1/utils.py @@ -0,0 +1,51 @@ + +""" +Course Experience API utilities. +""" +from eventtracking import tracker + +from lms.djangoapps.courseware.access import has_access +from lms.djangoapps.courseware.masquerade import is_masquerading, setup_masquerade +from lms.djangoapps.course_api.api import course_detail +from openedx.core.djangoapps.schedules.utils import reset_self_paced_schedule +from openedx.features.course_experience.utils import dates_banner_should_display + + +def reset_deadlines_for_course(request, course_key, research_event_data={}): # lint-amnesty, pylint: disable=dangerous-default-value + """ + Set the start_date of a schedule to today, which in turn will adjust due dates for + sequentials belonging to a self paced course + + Args: + request (Request): The request object + course_key (str): The course key + research_event_data (dict): Any data that should be included in the research tracking event + Example: sending the location of where the reset deadlines banner (i.e. outline-tab) + """ + + course_masquerade, user = setup_masquerade( + request, + course_key, + has_access(request.user, 'staff', course_key) + ) + + # We ignore the missed_deadlines because this util is used in endpoint from the Learning MFE for + # learners who have remaining attempts on a problem and reset their due dates in order to + # submit additional attempts. This can apply for 'completed' (submitted) content that would + # not be marked as past_due + _missed_deadlines, missed_gated_content = dates_banner_should_display(course_key, user) + if not missed_gated_content: + reset_self_paced_schedule(user, course_key) + + course_overview = course_detail(request, user.username, course_key) + # For context here, research_event_data should already contain `location` indicating + # the page/location dates were reset from and could also contain `block_id` if reset + # within courseware. + research_event_data.update({ + 'courserun_key': str(course_key), + 'is_masquerading': is_masquerading(user, course_key, course_masquerade), + 'is_staff': has_access(user, 'staff', course_key).has_access, + 'org_key': course_overview.display_org_with_default, + 'user_id': user.id, + }) + tracker.emit('edx.ui.lms.reset_deadlines.clicked', research_event_data) diff --git a/openedx/features/course_experience/api/v1/views.py b/openedx/features/course_experience/api/v1/views.py index 16be4a4e0e..10f8b8b4bf 100644 --- a/openedx/features/course_experience/api/v1/views.py +++ b/openedx/features/course_experience/api/v1/views.py @@ -5,7 +5,6 @@ import logging from django.utils.html import format_html from django.utils.translation import gettext as _ -from eventtracking import tracker from rest_framework.decorators import api_view, authentication_classes, permission_classes from rest_framework.exceptions import APIException, ParseError @@ -17,17 +16,14 @@ from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthenticat from edx_rest_framework_extensions.auth.session.authentication import SessionAuthenticationAllowInactiveUser from opaque_keys.edx.keys import CourseKey -from lms.djangoapps.course_api.api import course_detail +from common.djangoapps.student.models import CourseEnrollment from lms.djangoapps.course_goals.models import UserActivity -from lms.djangoapps.courseware.access import has_access from lms.djangoapps.courseware.courses import get_course_with_access -from lms.djangoapps.courseware.masquerade import is_masquerading, setup_masquerade -from openedx.core.djangoapps.schedules.utils import reset_self_paced_schedule from openedx.core.lib.api.authentication import BearerAuthenticationAllowInactiveUser from openedx.features.course_experience.api.v1.serializers import CourseDeadlinesMobileSerializer from openedx.features.course_experience.url_helpers import get_learning_mfe_home_url -from openedx.features.course_experience.utils import dates_banner_should_display +from openedx.features.course_experience.api.v1.utils import reset_deadlines_for_course log = logging.getLogger(__name__) @@ -65,32 +61,7 @@ def reset_course_deadlines(request): try: course_key = CourseKey.from_string(course_key) - course_masquerade, user = setup_masquerade( - request, - course_key, - has_access(request.user, 'staff', course_key) - ) - - # We ignore the missed_deadlines because this endpoint is used in the Learning MFE for - # learners who have remaining attempts on a problem and reset their due dates in order to - # submit additional attempts. This can apply for 'completed' (submitted) content that would - # not be marked as past_due - _missed_deadlines, missed_gated_content = dates_banner_should_display(course_key, user) - if not missed_gated_content: - reset_self_paced_schedule(user, course_key) - - course_overview = course_detail(request, user.username, course_key) - # For context here, research_event_data should already contain `location` indicating - # the page/location dates were reset from and could also contain `block_id` if reset - # within courseware. - research_event_data.update({ - 'courserun_key': str(course_key), - 'is_masquerading': is_masquerading(user, course_key, course_masquerade), - 'is_staff': has_access(user, 'staff', course_key).has_access, - 'org_key': course_overview.display_org_with_default, - 'user_id': user.id, - }) - tracker.emit('edx.ui.lms.reset_deadlines.clicked', research_event_data) + reset_deadlines_for_course(request, course_key, research_event_data) body_link = get_learning_mfe_home_url(course_key=course_key, url_fragment='dates') @@ -106,6 +77,52 @@ def reset_course_deadlines(request): raise UnableToResetDeadlines from reset_deadlines_exception +@api_view(["POST"]) +@authentication_classes( + ( + JwtAuthentication, + BearerAuthenticationAllowInactiveUser, + SessionAuthenticationAllowInactiveUser, + ) +) +@permission_classes((IsAuthenticated,)) +def reset_all_relative_course_deadlines(request): + """ + Set the start_date of a schedule to today for all enrolled courses + + Request Parameters: + research_event_data: any data that should be included in the research tracking event + Example: sending the location of where the reset deadlines banner (i.e. outline-tab) + + Returns: + success_course_keys: list of course keys for which deadlines were successfully reset + failed_course_keys: list of course keys for which deadlines could not be reset + """ + research_event_data = request.data.get("research_event_data", {}) + course_keys = ( + CourseEnrollment.enrollments_for_user(request.user).select_related("course").values_list("course_id", flat=True) + ) + + failed_course_keys = [] + success_course_keys = [] + + for course_key in course_keys: + try: + reset_deadlines_for_course(request, course_key, research_event_data) + success_course_keys.append(str(course_key)) + except Exception: # pylint: disable=broad-exception-caught + log.exception(f"Error occurred while trying to reset deadlines for course {course_key}!") + failed_course_keys.append(str(course_key)) + continue + + return Response( + { + "success_course_keys": success_course_keys, + "failed_course_keys": failed_course_keys, + } + ) + + class CourseDeadlinesMobileView(RetrieveAPIView): """ **Use Cases** From ae8996f68bcf4d029d28686e22b67ec10878ce1b Mon Sep 17 00:00:00 2001 From: awais qureshi Date: Tue, 16 Sep 2025 12:02:43 +0500 Subject: [PATCH 02/54] feat!: Upgrading to `django52`. --- requirements/constraints.txt | 4 -- requirements/edx-sandbox/base.txt | 6 +-- requirements/edx/base.txt | 25 ++++++----- requirements/edx/development.txt | 43 +++++++++---------- requirements/edx/doc.txt | 25 ++++++----- requirements/edx/semgrep.txt | 2 +- requirements/edx/testing.txt | 33 +++++++------- scripts/user_retirement/requirements/base.txt | 23 +++++----- .../user_retirement/requirements/testing.txt | 24 +++++------ 9 files changed, 88 insertions(+), 97 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index e0543146ff..4c95925bcc 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -19,10 +19,6 @@ # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35280 celery>=5.2.2,<6.0.0 -# Date: 2024-02-02 -# Stay on LTS version, remove once this is added to common constraint -Django<5.0 - # Date: 2020-02-10 # django-oauth-toolkit version >=2.0.0 has breaking changes. More details # mentioned on this issue https://github.com/openedx/edx-platform/issues/32884 diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index 4b5fd28677..d1bb7ca61c 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -60,9 +60,9 @@ packaging==25.0 # via matplotlib pillow==11.3.0 # via matplotlib -pycparser==2.22 +pycparser==2.23 # via cffi -pyparsing==3.2.3 +pyparsing==3.2.4 # via # -r requirements/edx-sandbox/base.in # chem @@ -74,7 +74,7 @@ random2==1.0.2 # via -r requirements/edx-sandbox/base.in regex==2025.9.1 # via nltk -scipy==1.16.1 +scipy==1.16.2 # via # -r requirements/edx-sandbox/base.in # chem diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index c28f8e68cd..e37dd0652e 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -70,14 +70,14 @@ bleach[css]==6.2.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/kernel.in -boto3==1.40.26 +boto3==1.40.31 # via # -r requirements/edx/kernel.in # django-ses # fs-s3fs # ora2 # snowflake-connector-python -botocore==1.40.26 +botocore==1.40.31 # via # -r requirements/edx/kernel.in # boto3 @@ -169,9 +169,8 @@ defusedxml==0.7.1 # ora2 # python3-openid # social-auth-core -django==4.2.24 +django==5.2.6 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in # django-appconf # django-autocomplete-light @@ -620,7 +619,7 @@ google-cloud-core==2.4.3 # google-cloud-storage google-cloud-firestore==2.21.0 # via firebase-admin -google-cloud-storage==3.3.1 +google-cloud-storage==3.4.0 # via firebase-admin google-crc32c==1.7.1 # via @@ -898,7 +897,7 @@ proto-plus==1.26.1 # via # google-api-core # google-cloud-firestore -protobuf==6.32.0 +protobuf==6.32.1 # via # google-api-core # google-cloud-firestore @@ -918,14 +917,14 @@ pyasn1-modules==0.4.2 # via google-auth pycountry==24.6.1 # via -r requirements/edx/kernel.in -pycparser==2.22 +pycparser==2.23 # via cffi pycryptodomex==3.23.0 # via # -r requirements/edx/kernel.in # edx-proctoring # lti-consumer-xblock -pydantic==2.11.7 +pydantic==2.11.9 # via camel-converter pydantic-core==2.33.2 # via pydantic @@ -956,15 +955,15 @@ pymongo==4.4.0 # event-tracking # mongoengine # openedx-forum -pynacl==1.5.0 +pynacl==1.6.0 # via # edx-django-utils # paramiko pynliner==0.8.0 # via -r requirements/edx/kernel.in -pyopenssl==25.1.0 +pyopenssl==25.2.0 # via snowflake-connector-python -pyparsing==3.2.3 +pyparsing==3.2.4 # via # chem # openedx-calc @@ -1082,11 +1081,11 @@ rules==3.5 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.13.1 +s3transfer==0.14.0 # via boto3 sailthru-client==2.2.3 # via edx-ace -scipy==1.16.1 +scipy==1.16.2 # via chem semantic-version==2.10.0 # via edx-drf-extensions diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 1096e5a455..5ec53a2b1d 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -140,7 +140,7 @@ boto==2.49.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -boto3==1.40.26 +boto3==1.40.31 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -148,7 +148,7 @@ boto3==1.40.26 # fs-s3fs # ora2 # snowflake-connector-python -botocore==1.40.26 +botocore==1.40.31 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -335,9 +335,8 @@ distlib==0.4.0 # via # -r requirements/edx/testing.txt # virtualenv -django==4.2.24 +django==5.2.6 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # django-appconf @@ -586,12 +585,12 @@ django-storages==1.14.6 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edxval -django-stubs[compatible-mypy]==5.2.2 +django-stubs[compatible-mypy]==5.2.5 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/development.in # djangorestframework-stubs -django-stubs-ext==5.2.2 +django-stubs-ext==5.2.5 # via django-stubs django-user-tasks==3.4.3 # via @@ -899,7 +898,7 @@ execnet==2.1.1 # pytest-xdist factory-boy==3.3.3 # via -r requirements/edx/testing.txt -faker==37.6.0 +faker==37.8.0 # via # -r requirements/edx/testing.txt # factory-boy @@ -985,7 +984,7 @@ google-cloud-firestore==2.21.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # firebase-admin -google-cloud-storage==3.3.1 +google-cloud-storage==3.4.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1086,7 +1085,7 @@ imagesize==1.4.1 # via # -r requirements/edx/doc.txt # sphinx -import-linter==2.4 +import-linter==2.5 # via -r requirements/edx/testing.txt importlib-metadata==8.7.0 # via @@ -1302,7 +1301,7 @@ multidict==6.6.4 # -r requirements/edx/testing.txt # aiohttp # yarl -mypy==1.17.1 +mypy==1.18.1 # via # -r requirements/edx/development.in # django-stubs @@ -1513,7 +1512,7 @@ proto-plus==1.26.1 # -r requirements/edx/testing.txt # google-api-core # google-cloud-firestore -protobuf==6.32.0 +protobuf==6.32.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1551,7 +1550,7 @@ pycountry==24.6.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -pycparser==2.22 +pycparser==2.23 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1562,7 +1561,7 @@ pycryptodomex==3.23.0 # -r requirements/edx/testing.txt # edx-proctoring # lti-consumer-xblock -pydantic==2.11.7 +pydantic==2.11.9 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1645,7 +1644,7 @@ pymongo==4.4.0 # event-tracking # mongoengine # openedx-forum -pynacl==1.5.0 +pynacl==1.6.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1655,12 +1654,12 @@ pynliner==0.8.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -pyopenssl==25.1.0 +pyopenssl==25.2.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # snowflake-connector-python -pyparsing==3.2.3 +pyparsing==3.2.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1700,7 +1699,7 @@ pytest==8.2.0 # pytest-xdist pytest-attrib==0.1.3 # via -r requirements/edx/testing.txt -pytest-cov==6.3.0 +pytest-cov==7.0.0 # via -r requirements/edx/testing.txt pytest-django==4.11.1 # via -r requirements/edx/testing.txt @@ -1710,7 +1709,7 @@ pytest-metadata==3.1.1 # via # -r requirements/edx/testing.txt # pytest-json-report -pytest-randomly==3.16.0 +pytest-randomly==4.0.1 # via -r requirements/edx/testing.txt pytest-xdist[psutil]==3.8.0 # via -r requirements/edx/testing.txt @@ -1870,7 +1869,7 @@ rules==3.5 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.13.1 +s3transfer==0.14.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1880,7 +1879,7 @@ sailthru-client==2.2.3 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-ace -scipy==1.16.1 +scipy==1.16.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2098,11 +2097,11 @@ tqdm==4.67.1 # -r requirements/edx/testing.txt # nltk # openai -types-pyyaml==6.0.12.20250822 +types-pyyaml==6.0.12.20250915 # via # django-stubs # djangorestframework-stubs -types-requests==2.32.4.20250809 +types-requests==2.32.4.20250913 # via djangorestframework-stubs typing-extensions==4.15.0 # via diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 13dc4c5c26..3a30e50ba4 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -105,14 +105,14 @@ bleach[css]==6.2.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/base.txt -boto3==1.40.26 +boto3==1.40.31 # via # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 # snowflake-connector-python -botocore==1.40.26 +botocore==1.40.31 # via # -r requirements/edx/base.txt # boto3 @@ -227,9 +227,8 @@ defusedxml==0.7.1 # ora2 # python3-openid # social-auth-core -django==4.2.24 +django==5.2.6 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # django-appconf # django-autocomplete-light @@ -725,7 +724,7 @@ google-cloud-firestore==2.21.0 # via # -r requirements/edx/base.txt # firebase-admin -google-cloud-storage==3.3.1 +google-cloud-storage==3.4.0 # via # -r requirements/edx/base.txt # firebase-admin @@ -1090,7 +1089,7 @@ proto-plus==1.26.1 # -r requirements/edx/base.txt # google-api-core # google-cloud-firestore -protobuf==6.32.0 +protobuf==6.32.1 # via # -r requirements/edx/base.txt # google-api-core @@ -1114,7 +1113,7 @@ pyasn1-modules==0.4.2 # google-auth pycountry==24.6.1 # via -r requirements/edx/base.txt -pycparser==2.22 +pycparser==2.23 # via # -r requirements/edx/base.txt # cffi @@ -1123,7 +1122,7 @@ pycryptodomex==3.23.0 # -r requirements/edx/base.txt # edx-proctoring # lti-consumer-xblock -pydantic==2.11.7 +pydantic==2.11.9 # via # -r requirements/edx/base.txt # camel-converter @@ -1168,18 +1167,18 @@ pymongo==4.4.0 # event-tracking # mongoengine # openedx-forum -pynacl==1.5.0 +pynacl==1.6.0 # via # -r requirements/edx/base.txt # edx-django-utils # paramiko pynliner==0.8.0 # via -r requirements/edx/base.txt -pyopenssl==25.1.0 +pyopenssl==25.2.0 # via # -r requirements/edx/base.txt # snowflake-connector-python -pyparsing==3.2.3 +pyparsing==3.2.4 # via # -r requirements/edx/base.txt # chem @@ -1318,7 +1317,7 @@ rules==3.5 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.13.1 +s3transfer==0.14.0 # via # -r requirements/edx/base.txt # boto3 @@ -1326,7 +1325,7 @@ sailthru-client==2.2.3 # via # -r requirements/edx/base.txt # edx-ace -scipy==1.16.1 +scipy==1.16.2 # via # -r requirements/edx/base.txt # chem diff --git a/requirements/edx/semgrep.txt b/requirements/edx/semgrep.txt index e5fe1dc79f..aec4c59acb 100644 --- a/requirements/edx/semgrep.txt +++ b/requirements/edx/semgrep.txt @@ -113,7 +113,7 @@ ruamel-yaml==0.18.15 # via semgrep ruamel-yaml-clib==0.2.12 # via ruamel-yaml -semgrep==1.135.0 +semgrep==1.136.0 # via -r requirements/edx/semgrep.in tomli==2.0.2 # via semgrep diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 29f63b9366..b9a53d6a2a 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -102,14 +102,14 @@ bleach[css]==6.2.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/base.txt -boto3==1.40.26 +boto3==1.40.31 # via # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 # snowflake-connector-python -botocore==1.40.26 +botocore==1.40.31 # via # -r requirements/edx/base.txt # boto3 @@ -253,9 +253,8 @@ dill==0.4.0 # via pylint distlib==0.4.0 # via virtualenv -django==4.2.24 +django==5.2.6 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # django-appconf # django-autocomplete-light @@ -694,7 +693,7 @@ execnet==2.1.1 # via pytest-xdist factory-boy==3.3.3 # via -r requirements/edx/testing.in -faker==37.6.0 +faker==37.8.0 # via factory-boy fastapi==0.116.1 # via pact-python @@ -756,7 +755,7 @@ google-cloud-firestore==2.21.0 # via # -r requirements/edx/base.txt # firebase-admin -google-cloud-storage==3.3.1 +google-cloud-storage==3.4.0 # via # -r requirements/edx/base.txt # firebase-admin @@ -831,7 +830,7 @@ idna==3.10 # requests # snowflake-connector-python # yarl -import-linter==2.4 +import-linter==2.5 # via -r requirements/edx/testing.in importlib-metadata==8.7.0 # via -r requirements/edx/base.txt @@ -1147,7 +1146,7 @@ proto-plus==1.26.1 # -r requirements/edx/base.txt # google-api-core # google-cloud-firestore -protobuf==6.32.0 +protobuf==6.32.1 # via # -r requirements/edx/base.txt # google-api-core @@ -1179,7 +1178,7 @@ pycodestyle==2.8.0 # -r requirements/edx/testing.in pycountry==24.6.1 # via -r requirements/edx/base.txt -pycparser==2.22 +pycparser==2.23 # via # -r requirements/edx/base.txt # cffi @@ -1188,7 +1187,7 @@ pycryptodomex==3.23.0 # -r requirements/edx/base.txt # edx-proctoring # lti-consumer-xblock -pydantic==2.11.7 +pydantic==2.11.9 # via # -r requirements/edx/base.txt # camel-converter @@ -1247,18 +1246,18 @@ pymongo==4.4.0 # event-tracking # mongoengine # openedx-forum -pynacl==1.5.0 +pynacl==1.6.0 # via # -r requirements/edx/base.txt # edx-django-utils # paramiko pynliner==0.8.0 # via -r requirements/edx/base.txt -pyopenssl==25.1.0 +pyopenssl==25.2.0 # via # -r requirements/edx/base.txt # snowflake-connector-python -pyparsing==3.2.3 +pyparsing==3.2.4 # via # -r requirements/edx/base.txt # chem @@ -1288,7 +1287,7 @@ pytest==8.2.0 # pytest-xdist pytest-attrib==0.1.3 # via -r requirements/edx/testing.in -pytest-cov==6.3.0 +pytest-cov==7.0.0 # via -r requirements/edx/testing.in pytest-django==4.11.1 # via -r requirements/edx/testing.in @@ -1298,7 +1297,7 @@ pytest-metadata==3.1.1 # via # -r requirements/edx/testing.in # pytest-json-report -pytest-randomly==3.16.0 +pytest-randomly==4.0.1 # via -r requirements/edx/testing.in pytest-xdist[psutil]==3.8.0 # via -r requirements/edx/testing.in @@ -1425,7 +1424,7 @@ rules==3.5 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.13.1 +s3transfer==0.14.0 # via # -r requirements/edx/base.txt # boto3 @@ -1433,7 +1432,7 @@ sailthru-client==2.2.3 # via # -r requirements/edx/base.txt # edx-ace -scipy==1.16.1 +scipy==1.16.2 # via # -r requirements/edx/base.txt # chem diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index 8b576aead5..205a449ffd 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -10,9 +10,9 @@ attrs==25.3.0 # via zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.in -boto3==1.40.26 +boto3==1.40.31 # via -r scripts/user_retirement/requirements/base.in -botocore==1.40.26 +botocore==1.40.31 # via # boto3 # s3transfer @@ -32,9 +32,8 @@ click==8.2.1 # edx-django-utils cryptography==45.0.7 # via pyjwt -django==4.2.24 +django==5.2.6 # via - # -c scripts/user_retirement/requirements/../../../requirements/constraints.txt # django-crum # django-waffle # edx-django-utils @@ -59,7 +58,7 @@ google-auth-httplib2==0.2.0 # via google-api-python-client googleapis-common-protos==1.70.0 # via google-api-core -httplib2==0.30.0 +httplib2==0.31.0 # via # google-api-python-client # google-auth-httplib2 @@ -67,7 +66,7 @@ idna==3.10 # via requests isodate==0.7.2 # via zeep -jenkinsapi==0.3.15 +jenkinsapi==0.3.16 # via -r scripts/user_retirement/requirements/base.in jmespath==1.0.1 # via @@ -83,7 +82,7 @@ platformdirs==4.4.0 # via zeep proto-plus==1.26.1 # via google-api-core -protobuf==6.32.0 +protobuf==6.32.1 # via # google-api-core # googleapis-common-protos @@ -96,15 +95,15 @@ pyasn1==0.6.1 # rsa pyasn1-modules==0.4.2 # via google-auth -pycparser==2.22 +pycparser==2.23 # via cffi pyjwt[crypto]==2.10.1 # via # edx-rest-api-client # simple-salesforce -pynacl==1.5.0 +pynacl==1.6.0 # via edx-django-utils -pyparsing==3.2.3 +pyparsing==3.2.4 # via httplib2 python-dateutil==2.9.0.post0 # via botocore @@ -130,7 +129,7 @@ requests-toolbelt==1.0.0 # via zeep rsa==4.9.1 # via google-auth -s3transfer==0.13.1 +s3transfer==0.14.0 # via boto3 simple-salesforce==1.12.9 # via -r scripts/user_retirement/requirements/base.in @@ -152,5 +151,5 @@ urllib3==2.5.0 # via # botocore # requests -zeep==4.3.1 +zeep==4.3.2 # via simple-salesforce diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index f5b98650f3..ff8bf18e61 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -14,11 +14,11 @@ attrs==25.3.0 # zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.txt -boto3==1.40.26 +boto3==1.40.31 # via # -r scripts/user_retirement/requirements/base.txt # moto -botocore==1.40.26 +botocore==1.40.31 # via # -r scripts/user_retirement/requirements/base.txt # boto3 @@ -52,7 +52,7 @@ cryptography==45.0.7 # pyjwt ddt==1.7.2 # via -r scripts/user_retirement/requirements/testing.in -django==4.2.24 +django==5.2.6 # via # -r scripts/user_retirement/requirements/base.txt # django-crum @@ -92,7 +92,7 @@ googleapis-common-protos==1.70.0 # via # -r scripts/user_retirement/requirements/base.txt # google-api-core -httplib2==0.30.0 +httplib2==0.31.0 # via # -r scripts/user_retirement/requirements/base.txt # google-api-python-client @@ -107,7 +107,7 @@ isodate==0.7.2 # via # -r scripts/user_retirement/requirements/base.txt # zeep -jenkinsapi==0.3.15 +jenkinsapi==0.3.16 # via -r scripts/user_retirement/requirements/base.txt jinja2==3.1.6 # via moto @@ -144,7 +144,7 @@ proto-plus==1.26.1 # via # -r scripts/user_retirement/requirements/base.txt # google-api-core -protobuf==6.32.0 +protobuf==6.32.1 # via # -r scripts/user_retirement/requirements/base.txt # google-api-core @@ -163,7 +163,7 @@ pyasn1-modules==0.4.2 # via # -r scripts/user_retirement/requirements/base.txt # google-auth -pycparser==2.22 +pycparser==2.23 # via # -r scripts/user_retirement/requirements/base.txt # cffi @@ -174,11 +174,11 @@ pyjwt[crypto]==2.10.1 # -r scripts/user_retirement/requirements/base.txt # edx-rest-api-client # simple-salesforce -pynacl==1.5.0 +pynacl==1.6.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils -pyparsing==3.2.3 +pyparsing==3.2.4 # via # -r scripts/user_retirement/requirements/base.txt # httplib2 @@ -229,7 +229,7 @@ rsa==4.9.1 # via # -r scripts/user_retirement/requirements/base.txt # google-auth -s3transfer==0.13.1 +s3transfer==0.14.0 # via # -r scripts/user_retirement/requirements/base.txt # boto3 @@ -268,9 +268,9 @@ urllib3==2.5.0 # responses werkzeug==3.1.3 # via moto -xmltodict==0.15.1 +xmltodict==1.0.0 # via moto -zeep==4.3.1 +zeep==4.3.2 # via # -r scripts/user_retirement/requirements/base.txt # simple-salesforce From b39e6ff20e80e1ecfb2879a3a3e7a0aefd24b70e Mon Sep 17 00:00:00 2001 From: Muhammad Arslan Abdul Rauf Date: Mon, 15 Sep 2025 21:06:03 +0500 Subject: [PATCH 03/54] fix: make ALLOWED_HOSTS configurable through YAML --- cms/envs/production.py | 19 ++++++++++++++----- lms/envs/production.py | 19 +++++++++++++++---- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/cms/envs/production.py b/cms/envs/production.py index f2d7ab88e4..09b203f6dd 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -89,6 +89,7 @@ with codecs.open(CONFIG_FILE, encoding='utf-8') as f: 'EVENT_BUS_PRODUCER_CONFIG', 'DEFAULT_FILE_STORAGE', 'STATICFILES_STORAGE', + 'ALLOWED_HOSTS', ] }) @@ -139,11 +140,19 @@ if STATIC_ROOT_BASE: DATA_DIR = path(DATA_DIR) -ALLOWED_HOSTS = [ - # TODO: bbeggs remove this before prod, temp fix to get load testing running - "*", - CMS_BASE, -] +# Configure ALLOWED_HOSTS based on YAML configuration +# If ALLOWED_HOSTS is explicitly set in YAML, use that; otherwise include "*" as fallback +if 'ALLOWED_HOSTS' in _YAML_TOKENS: + # User has explicitly configured ALLOWED_HOSTS in YAML + ALLOWED_HOSTS = _YAML_TOKENS['ALLOWED_HOSTS'] +else: + # Default behavior: include wildcard and CMS_BASE + ALLOWED_HOSTS = [ + "*", + ] + +if CMS_BASE and CMS_BASE not in ALLOWED_HOSTS: + ALLOWED_HOSTS.append(CMS_BASE) # Cache used for location mapping -- called many times with the same key/value # in a given request. diff --git a/lms/envs/production.py b/lms/envs/production.py index 0620d4f2c0..8f51250191 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -84,6 +84,7 @@ with codecs.open(CONFIG_FILE, encoding='utf-8') as f: 'EVENT_BUS_PRODUCER_CONFIG', 'DEFAULT_FILE_STORAGE', 'STATICFILES_STORAGE', + 'ALLOWED_HOSTS', ] }) @@ -141,10 +142,20 @@ SESSION_COOKIE_SAMESITE = DCS_SESSION_COOKIE_SAMESITE for feature, value in _YAML_TOKENS.get('FEATURES', {}).items(): FEATURES[feature] = value -ALLOWED_HOSTS = [ - "*", - _YAML_TOKENS.get('LMS_BASE'), -] +# Configure ALLOWED_HOSTS based on YAML configuration +# If ALLOWED_HOSTS is explicitly set in YAML, use that; otherwise include "*" as fallback +if 'ALLOWED_HOSTS' in _YAML_TOKENS: + # User has explicitly configured ALLOWED_HOSTS in YAML + ALLOWED_HOSTS = _YAML_TOKENS['ALLOWED_HOSTS'] +else: + # Default behavior: include wildcard and LMS_BASE + ALLOWED_HOSTS = [ + "*", + ] + +LMS_BASE = _YAML_TOKENS.get('LMS_BASE') +if LMS_BASE and LMS_BASE not in ALLOWED_HOSTS: + ALLOWED_HOSTS.append(LMS_BASE) # Cache used for location mapping -- called many times with the same key/value # in a given request. From 83dbf263d7d6bd881fb87577631059f6b7d0f601 Mon Sep 17 00:00:00 2001 From: Muhammad Arslan Abdul Rauf Date: Tue, 16 Sep 2025 23:31:01 +0500 Subject: [PATCH 04/54] refactor: move ALLOWED_HOSTS to openedx/envs/common --- cms/envs/production.py | 14 ++------------ lms/envs/production.py | 15 ++------------- 2 files changed, 4 insertions(+), 25 deletions(-) diff --git a/cms/envs/production.py b/cms/envs/production.py index 09b203f6dd..6e39cf02c0 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -140,19 +140,9 @@ if STATIC_ROOT_BASE: DATA_DIR = path(DATA_DIR) -# Configure ALLOWED_HOSTS based on YAML configuration -# If ALLOWED_HOSTS is explicitly set in YAML, use that; otherwise include "*" as fallback +# If ALLOWED_HOSTS is explicitly set in YAML, use it as the base; otherwise use default from common.py if 'ALLOWED_HOSTS' in _YAML_TOKENS: - # User has explicitly configured ALLOWED_HOSTS in YAML - ALLOWED_HOSTS = _YAML_TOKENS['ALLOWED_HOSTS'] -else: - # Default behavior: include wildcard and CMS_BASE - ALLOWED_HOSTS = [ - "*", - ] - -if CMS_BASE and CMS_BASE not in ALLOWED_HOSTS: - ALLOWED_HOSTS.append(CMS_BASE) + _BASE_ALLOWED_HOSTS = _YAML_TOKENS['ALLOWED_HOSTS'] # Cache used for location mapping -- called many times with the same key/value # in a given request. diff --git a/lms/envs/production.py b/lms/envs/production.py index 8f51250191..7e48a3c682 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -142,20 +142,9 @@ SESSION_COOKIE_SAMESITE = DCS_SESSION_COOKIE_SAMESITE for feature, value in _YAML_TOKENS.get('FEATURES', {}).items(): FEATURES[feature] = value -# Configure ALLOWED_HOSTS based on YAML configuration -# If ALLOWED_HOSTS is explicitly set in YAML, use that; otherwise include "*" as fallback +# If ALLOWED_HOSTS is explicitly set in YAML, use it as the base; otherwise use default from common.py if 'ALLOWED_HOSTS' in _YAML_TOKENS: - # User has explicitly configured ALLOWED_HOSTS in YAML - ALLOWED_HOSTS = _YAML_TOKENS['ALLOWED_HOSTS'] -else: - # Default behavior: include wildcard and LMS_BASE - ALLOWED_HOSTS = [ - "*", - ] - -LMS_BASE = _YAML_TOKENS.get('LMS_BASE') -if LMS_BASE and LMS_BASE not in ALLOWED_HOSTS: - ALLOWED_HOSTS.append(LMS_BASE) + _BASE_ALLOWED_HOSTS = _YAML_TOKENS['ALLOWED_HOSTS'] # Cache used for location mapping -- called many times with the same key/value # in a given request. From 245c76fc1bbf3bf30df537ebdc839ed71ff0a736 Mon Sep 17 00:00:00 2001 From: Muhammad Arslan Abdul Rauf Date: Mon, 22 Sep 2025 16:38:05 +0500 Subject: [PATCH 05/54] fix: add '*' wild card in common ALLOWED_HOSTS --- cms/envs/production.py | 5 ----- lms/envs/production.py | 5 ----- openedx/envs/common.py | 4 ++++ 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/cms/envs/production.py b/cms/envs/production.py index 6e39cf02c0..c6a0f090f3 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -89,7 +89,6 @@ with codecs.open(CONFIG_FILE, encoding='utf-8') as f: 'EVENT_BUS_PRODUCER_CONFIG', 'DEFAULT_FILE_STORAGE', 'STATICFILES_STORAGE', - 'ALLOWED_HOSTS', ] }) @@ -140,10 +139,6 @@ if STATIC_ROOT_BASE: DATA_DIR = path(DATA_DIR) -# If ALLOWED_HOSTS is explicitly set in YAML, use it as the base; otherwise use default from common.py -if 'ALLOWED_HOSTS' in _YAML_TOKENS: - _BASE_ALLOWED_HOSTS = _YAML_TOKENS['ALLOWED_HOSTS'] - # Cache used for location mapping -- called many times with the same key/value # in a given request. if 'loc_cache' not in CACHES: diff --git a/lms/envs/production.py b/lms/envs/production.py index 7e48a3c682..aeccaf0c0f 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -84,7 +84,6 @@ with codecs.open(CONFIG_FILE, encoding='utf-8') as f: 'EVENT_BUS_PRODUCER_CONFIG', 'DEFAULT_FILE_STORAGE', 'STATICFILES_STORAGE', - 'ALLOWED_HOSTS', ] }) @@ -142,10 +141,6 @@ SESSION_COOKIE_SAMESITE = DCS_SESSION_COOKIE_SAMESITE for feature, value in _YAML_TOKENS.get('FEATURES', {}).items(): FEATURES[feature] = value -# If ALLOWED_HOSTS is explicitly set in YAML, use it as the base; otherwise use default from common.py -if 'ALLOWED_HOSTS' in _YAML_TOKENS: - _BASE_ALLOWED_HOSTS = _YAML_TOKENS['ALLOWED_HOSTS'] - # Cache used for location mapping -- called many times with the same key/value # in a given request. if 'loc_cache' not in CACHES: diff --git a/openedx/envs/common.py b/openedx/envs/common.py index 18fa201382..e9033a1759 100644 --- a/openedx/envs/common.py +++ b/openedx/envs/common.py @@ -2258,6 +2258,10 @@ AI_TRANSLATIONS_API_URL = 'http://localhost:18760/api/v1' def should_send_learning_badge_events(settings): return settings.BADGES_ENABLED +############################## ALLOWED_HOSTS ############################### + +ALLOWED_HOSTS = ['*'] + ############################## Miscellaneous ############################### COURSE_MODE_DEFAULTS = { From 89d72074fd775ead87666f1bd63938fcc582d22d Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 22 Sep 2025 15:31:05 -0400 Subject: [PATCH 06/54] refactor: Move user_util library to edx-platform The library consisted of this set of utilities and a cli and was only being used in the edx-platform repo. The CLI will be DEPRed along with the repo but the code that is being used for retirement will be moved here. --- common/djangoapps/student/models/user.py | 2 +- lms/djangoapps/program_enrollments/models.py | 2 +- openedx/core/lib/tests/test_user_util.py | 241 +++++++++++++++++++ openedx/core/lib/user_util.py | 157 ++++++++++++ 4 files changed, 400 insertions(+), 2 deletions(-) create mode 100644 openedx/core/lib/tests/test_user_util.py create mode 100644 openedx/core/lib/user_util.py diff --git a/common/djangoapps/student/models/user.py b/common/djangoapps/student/models/user.py index 6d16a5a95b..8cd1fcef7a 100644 --- a/common/djangoapps/student/models/user.py +++ b/common/djangoapps/student/models/user.py @@ -44,7 +44,7 @@ from eventtracking import tracker from model_utils.models import TimeStampedModel from opaque_keys.edx.django.models import CourseKeyField, LearningContextKeyField from pytz import UTC, timezone -from user_util import user_util +from openedx.core.lib import user_util import openedx.core.djangoapps.django_comment_common.comment_client as cc from common.djangoapps.util.model_utils import emit_field_changed_events, get_changed_fields_dict diff --git a/lms/djangoapps/program_enrollments/models.py b/lms/djangoapps/program_enrollments/models.py index 04114a9dd5..2abec16d93 100644 --- a/lms/djangoapps/program_enrollments/models.py +++ b/lms/djangoapps/program_enrollments/models.py @@ -10,7 +10,7 @@ from django.utils.translation import gettext_lazy as _ from model_utils.models import TimeStampedModel from opaque_keys.edx.django.models import CourseKeyField from simple_history.models import HistoricalRecords -from user_util import user_util +from openedx.core.lib import user_util from common.djangoapps.student.models import CourseEnrollment diff --git a/openedx/core/lib/tests/test_user_util.py b/openedx/core/lib/tests/test_user_util.py new file mode 100644 index 0000000000..f269d8a221 --- /dev/null +++ b/openedx/core/lib/tests/test_user_util.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python + +"""Tests for `user_util` package.""" + +import pytest +from types import GeneratorType + +from openedx.core.lib import user_util + +VALID_SALT_LIST_ONE_SALT = ['gsw@&2p)$^p2hdk&ou0e%c=ou80o=%!+tv7(u(ircv@+96jl6$'] +VALID_SALT_LIST_THREE_SALTS = [ + '^==!0%=z4s!v7!yl0#+m6-st^*946aop6$0i+hu13&h_$a$vq8', + 'wdwhs@(f=jnlky4up8p0#04t$jp%ip)nfp@de6rr9i)j7nf', + ')h1^pu8a!rh=%$_4f7sx*5^46ln_pujw6y*s0=dl6i$_#&#io1', +] +VALID_SALT_LIST_FIVE_SALTS = [ + '8rv!7iy4a7mdvs_kudis6&oycj0_b(mj0s^@*e5p)(o+m(c-cb', + 'xp)43m+d_!f!-)c=ki_8oc2w9(^r^umy73%dp@z7sknn#800z$', + 'some_salt_that_is_not_very_random', + '$=ldtvagk$qwc)cz%2%edaa_id45^(xg*1rs#t0inywla*)3+x', + '4eyp*!%nz&g@8(tm!236ykbg2xzwcix!=)06q&=d2rh@3n1o+8', +] +VALID_SALT_LISTS = ( + VALID_SALT_LIST_ONE_SALT, + VALID_SALT_LIST_THREE_SALTS, + VALID_SALT_LIST_FIVE_SALTS, +) +INVALID_SALT_LIST = ( + 'gsw@&2p)$^p2hdk&ou0e%c=ou80o=%!+tv7(u(ircv@+96jl6$', + None, + [], +) + +# +# Username retirement tests +# + +@pytest.mark.parametrize('salt_list', VALID_SALT_LISTS) +def test_username_to_hash(salt_list): + username = 'ALearnerUserName' + retired_username = user_util.get_retired_username(username, salt_list) + assert retired_username != username + assert retired_username.startswith('_'.join(user_util.RETIRED_USERNAME_DEFAULT_FMT.split('_')[0:-1])) + # Since SHA1 is used, the hexadecimal digest length should be 40. + assert len(retired_username.split('_')[-1]) == 40 + + +@pytest.mark.parametrize('salt_list', VALID_SALT_LISTS) +def test_username_to_hash_is_normalized(salt_list): + """ + Make sure identical usernames with different cases map to the same retired username. + """ + username_mixed = 'ALearnerUserName' + username_lower = username_mixed.lower() + retired_username_mixed = user_util.get_retired_username(username_mixed, salt_list) + retired_username_lower = user_util.get_retired_username(username_lower, salt_list) + # No matter the case of the input username, the retired username hash should be identical. + assert retired_username_mixed == retired_username_lower + + +def test_unicode_username_to_hash(): + username = 'ÁĹéáŕńéŕŰśéŕŃáḿéẂíthŰńíćődé' + retired_username = user_util.get_retired_username(username, VALID_SALT_LIST_ONE_SALT) + assert retired_username != username + # Since SHA1 is used, the hexadecimal digest length should be 40. + assert len(retired_username.split('_')[-1]) == 40 + + +@pytest.mark.parametrize('salt_list', (VALID_SALT_LIST_THREE_SALTS,)) +def test_correct_username_hash(salt_list): + """ + Verify that get_retired_username uses the current salt and returns the expected hash. + """ + username = 'ALearnerUserName' + # Valid retired usernames for the above username when using VALID_SALT_LIST_THREE_SALTS. + valid_retired_usernames = [ + user_util.RETIRED_USERNAME_DEFAULT_FMT.format(user_util._compute_retired_hash(username.lower(), salt)) + for salt in salt_list + ] + retired_username = user_util.get_retired_username(username, salt_list) + assert retired_username == valid_retired_usernames[-1] + + +@pytest.mark.parametrize('salt_list', (VALID_SALT_LIST_FIVE_SALTS,)) +def test_all_usernames_to_hash(salt_list): + username = 'ALearnerUserName' + retired_username_generator = user_util.get_all_retired_usernames(username, salt_list) + assert isinstance(retired_username_generator, GeneratorType) + assert len(list(retired_username_generator)) == len(VALID_SALT_LIST_FIVE_SALTS) + + +@pytest.mark.parametrize('salt_list', VALID_SALT_LISTS) +def test_username_to_hash_with_different_format(salt_list): + username = 'ALearnerUserName' + retired_username_fmt = "{}_is_now_the_retired_username" + retired_username = user_util.get_retired_username(username, salt_list, retired_username_fmt=retired_username_fmt) + assert retired_username.endswith('_'.join(retired_username_fmt.split('_')[1:])) + # Since SHA1 is used, the hexadecimal digest length should be 40. + assert len(retired_username.split('_')[0]) == 40 + +# +# Email address retirement tests +# + +@pytest.mark.parametrize('salt_list', VALID_SALT_LISTS) +def test_email_to_hash(salt_list): + email = 'a.learner@example.com' + retired_email = user_util.get_retired_email(email, salt_list) + assert retired_email != email + assert retired_email.startswith('_'.join(user_util.RETIRED_EMAIL_DEFAULT_FMT.split('_')[0:2])) + assert retired_email.endswith(user_util.RETIRED_EMAIL_DEFAULT_FMT.split('@')[-1]) + # Since SHA1 is used, the hexadecimal digest length should be 40. + assert len(retired_email.split('@')[0]) == len('retired_email_') + 40 + + +@pytest.mark.parametrize('salt_list', VALID_SALT_LISTS) +def test_email_to_hash_is_normalized(salt_list): + """ + Make sure identical emails with different cases map to the same retired email. + """ + email_mixed = 'A.Learner@example.com' + email_lower = email_mixed.lower() + retired_email_mixed = user_util.get_retired_email(email_mixed, salt_list) + retired_email_lower = user_util.get_retired_email(email_lower, salt_list) + # No matter the case of the input email, the retired email hash should be identical. + assert retired_email_mixed == retired_email_lower + + +def test_unicode_email_to_hash(): + email = '🅐.🅛🅔🅐🅡🅝🅔🅡r@example.com' + retired_email = user_util.get_retired_email(email, VALID_SALT_LIST_ONE_SALT) + assert retired_email != email + # Since SHA1 is used, the hexadecimal digest length should be 40. + assert len(retired_email.split('@')[0]) == len('retired_email_') + 40 + + +@pytest.mark.parametrize('salt_list', (VALID_SALT_LIST_THREE_SALTS,)) +def test_correct_email_hash(salt_list): + """ + Verify that get_retired_email uses the current salt and returns the expected hash. + """ + email = 'a.learner@example.com' + # Valid retired emails for the above email address when using VALID_SALT_LIST_THREE_SALTS. + valid_retired_emails = [ + user_util.RETIRED_EMAIL_DEFAULT_FMT.format(user_util._compute_retired_hash(email.lower(), salt)) + for salt in salt_list + ] + retired_email = user_util.get_retired_email(email, salt_list) + assert retired_email == valid_retired_emails[-1] + + +@pytest.mark.parametrize('salt_list', (VALID_SALT_LIST_FIVE_SALTS,)) +def test_all_emails_to_hash(salt_list): + email = 'a.learner@example.com' + retired_email_generator = user_util.get_all_retired_emails(email, salt_list) + assert isinstance(retired_email_generator, GeneratorType) + assert len(list(retired_email_generator)) == len(VALID_SALT_LIST_FIVE_SALTS) + + +@pytest.mark.parametrize('salt_list', VALID_SALT_LISTS) +def test_email_to_hash_with_different_format(salt_list): + email = 'a.learner@example.com' + retired_email_fmt = "{}_is_now_the_retired_email@devnull.example.com" + retired_email = user_util.get_retired_email(email, salt_list, retired_email_fmt=retired_email_fmt) + assert retired_email.endswith('_'.join(retired_email_fmt.split('_')[1:])) + # Since SHA1 is used, the hexadecimal digest length should be 40. + assert len(retired_email.split('_')[0]) == 40 + +# +# Bad salt tests. +# + +@pytest.mark.parametrize('salt', INVALID_SALT_LIST) +def test_username_to_hash_bad_salt(salt): + """ + Salts that are *not* lists/tuples should fail. + """ + with pytest.raises((ValueError, IndexError)): + _ = user_util.get_retired_username('AnotherLearnerUserName', salt) + + +# +# External user retirement tests +# + +@pytest.mark.parametrize('salt_list', VALID_SALT_LISTS) +def test_external_key_to_hash(salt_list): + external_key = '343ni3hr3ifh3fgghg' + retired_external_key = user_util.get_retired_external_key(external_key, salt_list) + assert retired_external_key != external_key + assert retired_external_key.startswith( + '_'.join(user_util.RETIRED_EXTERNAL_KEY_DEFAULT_FMT.split('_')[0:3]) + ) + # Since SHA1 is used, the hexadecimal digest length should be 40. + assert len(retired_external_key) == len('retired_external_key_') + 40 + + +def test_unicode_external_key_to_hash(): + unicode_external_key = '🅐.🅛🅔🅐🅡🅝🅔🅡' + retired_external_key= user_util.get_retired_external_key(unicode_external_key, VALID_SALT_LIST_ONE_SALT) + assert retired_external_key != unicode_external_key + # Since SHA1 is used, the hexadecimal digest length should be 40. + assert len(retired_external_key) == len('retired_external_key_') + 40 + + +@pytest.mark.parametrize('salt_list', (VALID_SALT_LIST_THREE_SALTS,)) +def test_correct_external_key_hash(salt_list): + """ + Verify that get_retired_external_key uses the current salt and returns the expected hash. + """ + external_key = 'S34839GEF3' + valid_retired_external_keys = [ + user_util.RETIRED_EXTERNAL_KEY_DEFAULT_FMT.format( + user_util._compute_retired_hash(external_key.lower(), salt) + ) + for salt in salt_list + ] + retired_email = user_util.get_retired_external_key(external_key, salt_list) + assert retired_email == valid_retired_external_keys[-1] + + +@pytest.mark.parametrize('salt_list', (VALID_SALT_LIST_FIVE_SALTS,)) +def test_all_external_keys_to_hash(salt_list): + external_key = 'S34839GEF3' + retired_external_key_generator = user_util.get_all_retired_external_keys(external_key, salt_list) + assert isinstance(retired_external_key_generator, GeneratorType) + assert len(list(retired_external_key_generator)) == len(VALID_SALT_LIST_FIVE_SALTS) + + +@pytest.mark.parametrize('salt_list', VALID_SALT_LISTS) +def test_external_key_to_hash_with_different_format(salt_list): + external_key = 'S34839GEF3' + retired_external_key_fmt = "{}_is_now_the_retired_external_key" + retired_external_key = user_util.get_retired_external_key( + external_key, + salt_list, + retired_external_key_fmt=retired_external_key_fmt + ) + assert retired_external_key.endswith('_is_now_the_retired_external_key') + # Since SHA1 is used, the hexadecimal digest length should be 40. + assert len(retired_external_key.split('_')[0]) == 40 diff --git a/openedx/core/lib/user_util.py b/openedx/core/lib/user_util.py new file mode 100644 index 0000000000..c15a2bac14 --- /dev/null +++ b/openedx/core/lib/user_util.py @@ -0,0 +1,157 @@ +"""Main module.""" +import hashlib + + +RETIRED_USERNAME_DEFAULT_FMT = 'retired_username_{}' +RETIRED_EMAIL_DEFAULT_FMT = 'retired_email_{}@retired.edx.org' +RETIRED_EXTERNAL_KEY_DEFAULT_FMT = 'retired_external_key_{}' +SALT_LIST_EXCEPTION = ValueError("Salt must be a list -or- tuple of all historical salts.") + + +def _compute_retired_hash(value_to_retire, salt): + """ + Returns a retired value given a value to retire and a hash. + + Arguments: + value_to_retire (str): Value to be retired. + salt (str): Salt string used to modify the retired value before hashing. + """ + return hashlib.sha1( + salt.encode() + value_to_retire.encode('utf-8') + ).hexdigest() + + +def get_all_retired_usernames(username, salt_list, retired_username_fmt=RETIRED_USERNAME_DEFAULT_FMT): + """ + Returns a generator of possible retired usernames based on the original + lowercased username and all the historical salts, from oldest to current. + The current salt is assumed to be the last salt in the list. + + Raises :class:`~ValueError` if the salt isn't a list of salts. + + Arguments: + username (str): The name of the user to be retired. + salt_list (list/tuple): List of all historical salts. + + Yields: + Returns a generator of possible retired usernames based on the original username + and all the historical salts, including the current salt, from oldest to current. + """ + if not isinstance(salt_list, (list, tuple)): + raise SALT_LIST_EXCEPTION + + for salt in salt_list: + yield retired_username_fmt.format(_compute_retired_hash(username.lower(), salt)) + + +def get_all_retired_emails(email, salt_list, retired_email_fmt=RETIRED_EMAIL_DEFAULT_FMT): + """ + Returns a generator of possible retired email addresses based on the + original lowercased email and all the historical salts, from oldest to + current. The current salt is assumed to be the last salt in the list. + + Raises :class:`~ValueError` if the salt isn't a list of salts. + + Arguments: + email (str): Email address of the user to be retired. + salt_list (list/tuple): List of all historical salts. + + Yields: + Returns a generator of possible retired email addresses based on the original email + and all the historical salts, including the current salt, from oldest to current. + """ + if not isinstance(salt_list, (list, tuple)): + raise SALT_LIST_EXCEPTION + + for salt in salt_list: + yield retired_email_fmt.format(_compute_retired_hash(email.lower(), salt)) + + +def get_all_retired_external_keys(external_key, salt_list, retired_external_key_fmt=RETIRED_EXTERNAL_KEY_DEFAULT_FMT): + """ + Returns a generator of possible retired external user key based on the + original external user key and all the historical salts, from oldest to + current. The current salt is assumed to be the last salt in the list. + + Raises :class:`~ValueError` if the salt isn't a list of salts. + + Arguments: + external_key (str): External user key of the user to be retired. + salt_list (list/tuple): List of all historical salts. + + Yields: + Returns a generator of possible retired external user keys based on the original external key + and all the historical salts, including the current salt, from oldest to current. + """ + if not isinstance(salt_list, (list, tuple)): + raise SALT_LIST_EXCEPTION + + for salt in salt_list: + yield retired_external_key_fmt.format(_compute_retired_hash(external_key.lower(), salt)) + + +def get_retired_username(username, salt_list, retired_username_fmt=RETIRED_USERNAME_DEFAULT_FMT): + """ + Returns a retired username based on the original lowercased username and + all the historical salts, from oldest to current. The current salt is + assumed to be the last salt in the list. + + Raises :class:`~ValueError` if the salt isn't a list of salts. + + Arguments: + username (str): The name of the user to be retired. + salt_list (list/tuple): List of all historical salts. + + Yields: + Returns a retired username based on the original username + and all the historical salts, including the current salt. + """ + if not isinstance(salt_list, (list, tuple)): + raise SALT_LIST_EXCEPTION + + return retired_username_fmt.format(_compute_retired_hash(username.lower(), salt_list[-1])) + + +def get_retired_email(email, salt_list, retired_email_fmt=RETIRED_EMAIL_DEFAULT_FMT): + """ + Returns a retired email address based on the original lowercased email + address and the current salt. The current salt is assumed to be the last + salt in the list. + + Raises :class:`~ValueError` if salt_list isn't a list of salts. + + Arguments: + email (str): Email address of the user to be retired. + salt_list (list/tuple): List of all historical salts. + + Yields: + Returns a retired email address based on the original email + and the current salt + """ + if not isinstance(salt_list, (list, tuple)): + raise SALT_LIST_EXCEPTION + + return retired_email_fmt.format(_compute_retired_hash(email.lower(), salt_list[-1])) + + +def get_retired_external_key(external_key, salt_list, retired_external_key_fmt=RETIRED_EXTERNAL_KEY_DEFAULT_FMT): + """ + Returns a retired external user key based on the original external key and the current salt. + The current salt is assumed to be the last salt in the list. + + Raises :class:`~ValueError` if salt_list isn't a list of salts. + + Arguments: + external_key (str): External user key of the user to be retired. + salt_list (list/tuple): List of all historical salts. + + Yields: + Returns a retired external user key based on the original external_user_key + and the current salt + """ + if not isinstance(salt_list, (list, tuple)): + raise SALT_LIST_EXCEPTION + + return retired_external_key_fmt.format( + _compute_retired_hash(external_key.lower(), salt_list[-1]) + ) From 70391fcc6efc8927d548f6a5268d1f9194312494 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 22 Sep 2025 15:40:47 -0400 Subject: [PATCH 07/54] build: Drop the user-util dependency. --- requirements/edx/kernel.in | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in index 043c8f4794..2b043f71dd 100644 --- a/requirements/edx/kernel.in +++ b/requirements/edx/kernel.in @@ -155,7 +155,6 @@ sorl-thumbnail sortedcontainers # Provides SortedKeyList, used for lists of XBlock assets stevedore # Support for runtime plugins, used for XBlocks and edx-platform Django app plugins unicodecsv # Easier support for CSV files with unicode text -user-util # Functionality for retiring users (GDPR compliance) webob web-fragments # Provides the ability to render fragments of web pages wrapt # Better functools.wrapped. TODO: functools has since improved, maybe we can switch? From 3cff8a5c0a2ac7d30f21dd66db6bec34336fa868 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 22 Sep 2025 15:43:37 -0400 Subject: [PATCH 08/54] chore: Run `make upgrade` --- requirements/edx/base.txt | 3 --- requirements/edx/development.txt | 5 ----- requirements/edx/doc.txt | 3 --- requirements/edx/testing.txt | 3 --- 4 files changed, 14 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 7d3799320a..584766f239 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -131,7 +131,6 @@ click==8.2.1 # code-annotations # edx-django-utils # nltk - # user-util click-didyoumean==0.3.1 # via celery click-plugins==1.1.1.2 @@ -1208,8 +1207,6 @@ urllib3==2.5.0 # botocore # elasticsearch # requests -user-util==2.0.0 - # via -r requirements/edx/kernel.in vine==5.1.0 # via # amqp diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index c96d8bbe19..2629f28e48 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -241,7 +241,6 @@ click==8.2.1 # nltk # pact-python # pip-tools - # user-util # uvicorn click-didyoumean==0.3.1 # via @@ -2160,10 +2159,6 @@ urllib3==2.5.0 # elasticsearch # requests # types-requests -user-util==2.0.0 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt uvicorn==0.35.0 # via # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 9618446f16..69a1a2845c 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -177,7 +177,6 @@ click==8.2.1 # code-annotations # edx-django-utils # nltk - # user-util click-didyoumean==0.3.1 # via # -r requirements/edx/base.txt @@ -1524,8 +1523,6 @@ urllib3==2.5.0 # botocore # elasticsearch # requests -user-util==2.0.0 - # via -r requirements/edx/base.txt vine==5.1.0 # via # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 5bd4e49957..c4f9d4091b 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -183,7 +183,6 @@ click==8.2.1 # import-linter # nltk # pact-python - # user-util # uvicorn click-didyoumean==0.3.1 # via @@ -1602,8 +1601,6 @@ urllib3==2.5.0 # botocore # elasticsearch # requests -user-util==2.0.0 - # via -r requirements/edx/base.txt uvicorn==0.35.0 # via pact-python vine==5.1.0 From 8a7665697ad58b8c049b12291085cce7d9530861 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 22 Sep 2025 15:55:17 -0400 Subject: [PATCH 09/54] fixup! refactor: Move user_util library to edx-platform --- openedx/core/lib/tests/test_user_util.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/openedx/core/lib/tests/test_user_util.py b/openedx/core/lib/tests/test_user_util.py index f269d8a221..48d7941aa0 100644 --- a/openedx/core/lib/tests/test_user_util.py +++ b/openedx/core/lib/tests/test_user_util.py @@ -31,10 +31,10 @@ INVALID_SALT_LIST = ( [], ) + # # Username retirement tests # - @pytest.mark.parametrize('salt_list', VALID_SALT_LISTS) def test_username_to_hash(salt_list): username = 'ALearnerUserName' @@ -74,6 +74,7 @@ def test_correct_username_hash(salt_list): username = 'ALearnerUserName' # Valid retired usernames for the above username when using VALID_SALT_LIST_THREE_SALTS. valid_retired_usernames = [ + # pylint: disable=protected-access user_util.RETIRED_USERNAME_DEFAULT_FMT.format(user_util._compute_retired_hash(username.lower(), salt)) for salt in salt_list ] @@ -98,10 +99,10 @@ def test_username_to_hash_with_different_format(salt_list): # Since SHA1 is used, the hexadecimal digest length should be 40. assert len(retired_username.split('_')[0]) == 40 + # # Email address retirement tests # - @pytest.mark.parametrize('salt_list', VALID_SALT_LISTS) def test_email_to_hash(salt_list): email = 'a.learner@example.com' @@ -142,6 +143,7 @@ def test_correct_email_hash(salt_list): email = 'a.learner@example.com' # Valid retired emails for the above email address when using VALID_SALT_LIST_THREE_SALTS. valid_retired_emails = [ + # pylint: disable=protected-access user_util.RETIRED_EMAIL_DEFAULT_FMT.format(user_util._compute_retired_hash(email.lower(), salt)) for salt in salt_list ] @@ -166,10 +168,10 @@ def test_email_to_hash_with_different_format(salt_list): # Since SHA1 is used, the hexadecimal digest length should be 40. assert len(retired_email.split('_')[0]) == 40 + # # Bad salt tests. # - @pytest.mark.parametrize('salt', INVALID_SALT_LIST) def test_username_to_hash_bad_salt(salt): """ @@ -197,7 +199,7 @@ def test_external_key_to_hash(salt_list): def test_unicode_external_key_to_hash(): unicode_external_key = '🅐.🅛🅔🅐🅡🅝🅔🅡' - retired_external_key= user_util.get_retired_external_key(unicode_external_key, VALID_SALT_LIST_ONE_SALT) + retired_external_key = user_util.get_retired_external_key(unicode_external_key, VALID_SALT_LIST_ONE_SALT) assert retired_external_key != unicode_external_key # Since SHA1 is used, the hexadecimal digest length should be 40. assert len(retired_external_key) == len('retired_external_key_') + 40 @@ -210,6 +212,7 @@ def test_correct_external_key_hash(salt_list): """ external_key = 'S34839GEF3' valid_retired_external_keys = [ + # pylint: disable=protected-access user_util.RETIRED_EXTERNAL_KEY_DEFAULT_FMT.format( user_util._compute_retired_hash(external_key.lower(), salt) ) From 990f29f9067844712be97db2318a8bc200e482c4 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Wed, 24 Sep 2025 16:29:56 +0500 Subject: [PATCH 10/54] chore: Run unit tests only pinned which is 5.2 Removed Django version '5.2' from the workflow. --- .github/workflows/unit-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 81f65eda77..2a2669bc68 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -22,7 +22,6 @@ jobs: - "3.11" django-version: - "pinned" - - "5.2" # When updating the shards, remember to make the same changes in # .github/workflows/unit-tests-gh-hosted.yml shard_name: From c2e4bbdde9eb81ffa2b933d430a694fee78bf5c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Sep 2025 04:19:07 +0000 Subject: [PATCH 11/54] chore(deps): bump actions/setup-node from 4 to 5 Bumps [actions/setup-node](https://github.com/actions/setup-node) from 4 to 5. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-node dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/js-tests.yml | 2 +- .github/workflows/quality-checks.yml | 2 +- .github/workflows/static-assets-check.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 94a1368e96..9252a823eb 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -23,7 +23,7 @@ jobs: run: git fetch --depth=1 origin master - name: Setup Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v5 with: node-version: ${{ matrix.node-version }} cache: 'npm' diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 3f4cbeeb4d..964e05b061 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -35,7 +35,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Setup Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v5 with: node-version: ${{ matrix.node-version }} diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index 43cb597c16..d78d67d583 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -48,7 +48,7 @@ jobs: sudo apt-get install libxmlsec1-dev pkg-config - name: Setup Node - uses: actions/setup-node@v4 + uses: actions/setup-node@v5 with: node-version: ${{ matrix.node-version }} From d87ba4764327801163b5035b15f601a33b8c29f8 Mon Sep 17 00:00:00 2001 From: Deimer M Date: Wed, 1 Oct 2025 12:10:58 -0500 Subject: [PATCH 12/54] fix: Profile MFE redirection issue (URL path override) This error was occurring because the way the redirect URL was constructed caused the entire base path to be removed. This commit updates the URL construction method to correctly preserve the MFE's path. --- lms/templates/header/user_dropdown.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/templates/header/user_dropdown.html b/lms/templates/header/user_dropdown.html index b4b22e0e32..a489c17412 100644 --- a/lms/templates/header/user_dropdown.html +++ b/lms/templates/header/user_dropdown.html @@ -44,7 +44,7 @@ should_show_order_history = not enterprise_customer_portal
${_("Dashboard")}
% endif -
${_("Profile")}
+
${_("Profile")}
${_("Account")}
% if should_show_order_history:
${_("Order History")}
From 3c31cdb9ebbf092c1d731e79c79b1290e4d8a032 Mon Sep 17 00:00:00 2001 From: Kyrylo Kholodenko Date: Fri, 19 Sep 2025 15:03:06 +0300 Subject: [PATCH 13/54] refactor: rename and refactor functions and view --- .../api/v1/tests/test_utils.py | 35 +++++++- .../api/v1/tests/test_views.py | 21 +++-- .../features/course_experience/api/v1/urls.py | 8 +- .../course_experience/api/v1/utils.py | 80 +++++++++++++++++-- .../course_experience/api/v1/views.py | 24 ++---- 5 files changed, 127 insertions(+), 41 deletions(-) diff --git a/openedx/features/course_experience/api/v1/tests/test_utils.py b/openedx/features/course_experience/api/v1/tests/test_utils.py index 10b5882918..741a2d7658 100644 --- a/openedx/features/course_experience/api/v1/tests/test_utils.py +++ b/openedx/features/course_experience/api/v1/tests/test_utils.py @@ -13,7 +13,11 @@ from common.djangoapps.util.testing import EventTestMixin from lms.djangoapps.course_home_api.tests.utils import BaseCourseHomeTests from lms.djangoapps.courseware.tests.helpers import MasqueradeMixin from openedx.core.djangoapps.schedules.models import Schedule -from openedx.features.course_experience.api.v1.utils import reset_deadlines_for_course +from openedx.features.course_experience.api.v1.utils import ( + reset_deadlines_for_course, + reset_course_deadlines_for_user, + reset_bulk_course_deadlines +) from xmodule.modulestore.tests.factories import CourseFactory @@ -82,3 +86,32 @@ class TestResetDeadlinesForCourse(EventTestMixin, BaseCourseHomeTests, Masquerad org_key=self.course.org, user_id=student_user_id, ) + + def test_reset_course_deadlines_for_user(self): + """Test the reset_course_deadlines_for_user utility function directly""" + enrollment = CourseEnrollment.enroll(self.user, self.course.id, CourseMode.VERIFIED) + enrollment.schedule.start_date = timezone.now() - datetime.timedelta(days=100) + enrollment.schedule.save() + + result = reset_course_deadlines_for_user(self.user, self.course.id) + + assert result is True + assert enrollment.schedule.start_date < Schedule.objects.get(id=enrollment.schedule.id).start_date + + def test_reset_bulk_course_deadlines(self): + """Test the reset_bulk_course_deadlines utility function""" + enrollment = CourseEnrollment.enroll(self.user, self.course.id, CourseMode.VERIFIED) + enrollment.schedule.start_date = timezone.now() - datetime.timedelta(days=100) + enrollment.schedule.save() + + request = APIRequestFactory().post( + reverse("course-experience-reset-all-course-deadlines"), {} + ) + request.user = self.user + + success_keys, failed_keys = reset_bulk_course_deadlines(request, [self.course.id], {}) + + assert len(success_keys) == 1 + assert self.course.id in success_keys + assert len(failed_keys) == 0 + assert enrollment.schedule.start_date < Schedule.objects.get(id=enrollment.schedule.id).start_date diff --git a/openedx/features/course_experience/api/v1/tests/test_views.py b/openedx/features/course_experience/api/v1/tests/test_views.py index d7e8f6cafc..097eb2c18c 100644 --- a/openedx/features/course_experience/api/v1/tests/test_views.py +++ b/openedx/features/course_experience/api/v1/tests/test_views.py @@ -89,30 +89,27 @@ class ResetAllRelativeCourseDeadlinesViewTests(BaseCourseHomeTests, MasqueradeMi self.enrollment.schedule.start_date = timezone.now() - datetime.timedelta(days=100) self.enrollment.schedule.save() - def test_reset_all_relative_course_deadlines(self): + def test_reset_all_course_deadlines(self): """ - Test reset all relative course deadlines endpoint + Test reset all course deadlines endpoint """ response = self.client.post( - reverse("course-experience-reset-all-relative-course-deadlines"), + reverse("course-experience-reset-all-course-deadlines"), {}, ) assert response.status_code == 200 assert self.enrollment.schedule.start_date < Schedule.objects.get(id=self.enrollment.schedule.id).start_date assert str(self.course.id) in response.data.get("success_course_keys") - def test_reset_all_relative_course_deadlines_failure(self): + def test_reset_all_course_deadlines_failure(self): """ - Raise exception on reset_deadlines_for_course and assert if failure course id is returned + Raise exception on reset_bulk_course_deadlines and assert if failure course id is returned """ with mock.patch( - "openedx.features.course_experience.api.v1.views.reset_deadlines_for_course", - side_effect=Exception("Test Exception"), + "openedx.features.course_experience.api.v1.views.reset_bulk_course_deadlines", + return_value=([], [self.course.id]), ): - response = self.client.post( - reverse("course-experience-reset-all-relative-course-deadlines"), - {}, - ) + response = self.client.post(reverse("course-experience-reset-all-course-deadlines"), {}) assert response.status_code == 200 assert str(self.course.id) in response.data.get("failed_course_keys") @@ -123,7 +120,7 @@ class ResetAllRelativeCourseDeadlinesViewTests(BaseCourseHomeTests, MasqueradeMi """ self.client.logout() response = self.client.post( - reverse("course-experience-reset-all-relative-course-deadlines"), + reverse("course-experience-reset-all-course-deadlines"), {}, ) assert response.status_code == 401 diff --git a/openedx/features/course_experience/api/v1/urls.py b/openedx/features/course_experience/api/v1/urls.py index 7e5a0936c2..30d7c55d29 100644 --- a/openedx/features/course_experience/api/v1/urls.py +++ b/openedx/features/course_experience/api/v1/urls.py @@ -8,7 +8,7 @@ from django.urls import re_path from openedx.features.course_experience.api.v1.views import ( reset_course_deadlines, - reset_all_relative_course_deadlines, + reset_all_course_deadlines, CourseDeadlinesMobileView, ) @@ -22,9 +22,9 @@ urlpatterns += [ name='course-experience-reset-course-deadlines' ), re_path( - r'v1/reset_all_relative_course_deadlines/', - reset_all_relative_course_deadlines, - name='course-experience-reset-all-relative-course-deadlines', + r'v1/reset_all_course_deadlines/', + reset_all_course_deadlines, + name='course-experience-reset-all-course-deadlines', ) ] diff --git a/openedx/features/course_experience/api/v1/utils.py b/openedx/features/course_experience/api/v1/utils.py index aabfadb540..8f9205b0f1 100644 --- a/openedx/features/course_experience/api/v1/utils.py +++ b/openedx/features/course_experience/api/v1/utils.py @@ -2,6 +2,7 @@ """ Course Experience API utilities. """ +import logging from eventtracking import tracker from lms.djangoapps.courseware.access import has_access @@ -11,6 +12,76 @@ from openedx.core.djangoapps.schedules.utils import reset_self_paced_schedule from openedx.features.course_experience.utils import dates_banner_should_display +logger = logging.getLogger(__name__) + + +def reset_course_deadlines_for_user(user, course_key): + """ + Core function to reset deadlines for a single course and user. + + Args: + user: The user object + course_key: The course key + + Returns: + bool: True if deadlines were reset, False if gated content prevents reset + """ + # We ignore the missed_deadlines because this util is used in endpoint from the Learning MFE for + # learners who have remaining attempts on a problem and reset their due dates in order to + # submit additional attempts. This can apply for 'completed' (submitted) content that would + # not be marked as past_due + _missed_deadlines, missed_gated_content = dates_banner_should_display(course_key, user) + if not missed_gated_content: + reset_self_paced_schedule(user, course_key) + return True + return False + + +def reset_bulk_course_deadlines(request, course_keys, research_event_data={}): # lint-amnesty, pylint: disable=dangerous-default-value + """ + Reset deadlines for multiple courses for the requesting user. + + Args: + request (Request): The request object + course_keys (list): List of course keys + research_event_data (dict): Any data that should be included in the research tracking event + + Returns: + tuple: (success_course_keys, failed_course_keys) + """ + success_course_keys = [] + failed_course_keys = [] + + for course_key in course_keys: + try: + course_masquerade, user = setup_masquerade( + request, + course_key, + has_access(request.user, 'staff', course_key) + ) + + if reset_course_deadlines_for_user(user, course_key): + success_course_keys.append(course_key) + + course_overview = course_detail(request, user.username, course_key) + + research_event_data.update({ + 'courserun_key': str(course_key), + 'is_masquerading': is_masquerading(user, course_key, course_masquerade), + 'is_staff': has_access(user, 'staff', course_key).has_access, + 'org_key': course_overview.display_org_with_default, + 'user_id': user.id, + }) + tracker.emit('edx.ui.lms.reset_deadlines.clicked', research_event_data) + else: + failed_course_keys.append(course_key) + except Exception: # pylint: disable=broad-exception-caught + logger.exception('Error occurred while trying to reset deadlines!') + failed_course_keys.append(course_key) + + return success_course_keys, failed_course_keys + + def reset_deadlines_for_course(request, course_key, research_event_data={}): # lint-amnesty, pylint: disable=dangerous-default-value """ Set the start_date of a schedule to today, which in turn will adjust due dates for @@ -29,14 +100,7 @@ def reset_deadlines_for_course(request, course_key, research_event_data={}): # has_access(request.user, 'staff', course_key) ) - # We ignore the missed_deadlines because this util is used in endpoint from the Learning MFE for - # learners who have remaining attempts on a problem and reset their due dates in order to - # submit additional attempts. This can apply for 'completed' (submitted) content that would - # not be marked as past_due - _missed_deadlines, missed_gated_content = dates_banner_should_display(course_key, user) - if not missed_gated_content: - reset_self_paced_schedule(user, course_key) - + if reset_course_deadlines_for_user(user, course_key): course_overview = course_detail(request, user.username, course_key) # For context here, research_event_data should already contain `location` indicating # the page/location dates were reset from and could also contain `block_id` if reset diff --git a/openedx/features/course_experience/api/v1/views.py b/openedx/features/course_experience/api/v1/views.py index 10f8b8b4bf..d822fdd65c 100644 --- a/openedx/features/course_experience/api/v1/views.py +++ b/openedx/features/course_experience/api/v1/views.py @@ -23,7 +23,7 @@ from lms.djangoapps.courseware.courses import get_course_with_access from openedx.core.lib.api.authentication import BearerAuthenticationAllowInactiveUser from openedx.features.course_experience.api.v1.serializers import CourseDeadlinesMobileSerializer from openedx.features.course_experience.url_helpers import get_learning_mfe_home_url -from openedx.features.course_experience.api.v1.utils import reset_deadlines_for_course +from openedx.features.course_experience.api.v1.utils import reset_deadlines_for_course, reset_bulk_course_deadlines log = logging.getLogger(__name__) @@ -86,7 +86,7 @@ def reset_course_deadlines(request): ) ) @permission_classes((IsAuthenticated,)) -def reset_all_relative_course_deadlines(request): +def reset_all_course_deadlines(request): """ Set the start_date of a schedule to today for all enrolled courses @@ -99,26 +99,18 @@ def reset_all_relative_course_deadlines(request): failed_course_keys: list of course keys for which deadlines could not be reset """ research_event_data = request.data.get("research_event_data", {}) - course_keys = ( + course_keys = list( CourseEnrollment.enrollments_for_user(request.user).select_related("course").values_list("course_id", flat=True) ) - failed_course_keys = [] - success_course_keys = [] - - for course_key in course_keys: - try: - reset_deadlines_for_course(request, course_key, research_event_data) - success_course_keys.append(str(course_key)) - except Exception: # pylint: disable=broad-exception-caught - log.exception(f"Error occurred while trying to reset deadlines for course {course_key}!") - failed_course_keys.append(str(course_key)) - continue + success_course_keys, failed_course_keys = reset_bulk_course_deadlines( + request, course_keys, research_event_data + ) return Response( { - "success_course_keys": success_course_keys, - "failed_course_keys": failed_course_keys, + "success_course_keys": [str(key) for key in success_course_keys], + "failed_course_keys": [str(key) for key in failed_course_keys], } ) From 4090e41f51220bc9bb42e774dc5df86df7367020 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Fri, 3 Oct 2025 10:17:19 -0400 Subject: [PATCH 14/54] build: Drop unused docker compose and sql files. --- .../workflows/docker-compose.yml.mysqldbdump | 23 ------------------- .github/workflows/init/01.sql | 3 --- 2 files changed, 26 deletions(-) delete mode 100644 .github/workflows/docker-compose.yml.mysqldbdump delete mode 100644 .github/workflows/init/01.sql diff --git a/.github/workflows/docker-compose.yml.mysqldbdump b/.github/workflows/docker-compose.yml.mysqldbdump deleted file mode 100644 index 87f0321374..0000000000 --- a/.github/workflows/docker-compose.yml.mysqldbdump +++ /dev/null @@ -1,23 +0,0 @@ -version: '3' -services: - mysql: - image: mysql:5.7 - container_name: edx.devstack.mysql80 - ports: - - '3306:3306' - environment: - MYSQL_ROOT_PASSWORD: "" - MYSQL_ALLOW_EMPTY_PASSWORD: "yes" - volumes: - - ./init:/docker-entrypoint-initdb.d - healthcheck: - test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] - timeout: 20s - retries: 10 - edxapp: - image: edxops/edxapp:latest - command: bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate' - volumes: - - ../../:/edx/app/edxapp/edx-platform - depends_on: - - mysql diff --git a/.github/workflows/init/01.sql b/.github/workflows/init/01.sql deleted file mode 100644 index 93d3a107e3..0000000000 --- a/.github/workflows/init/01.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE DATABASE IF NOT EXISTS `edxapp`; -CREATE DATABASE IF NOT EXISTS `edxapp_csmh`; -GRANT ALL PRIVILEGES ON *.* TO 'edxapp001'@'%' IDENTIFIED BY 'password'; From 154c224e541b75a648ceccf4f681379d0a95776d Mon Sep 17 00:00:00 2001 From: sameeramin <35958006+sameeramin@users.noreply.github.com> Date: Fri, 3 Oct 2025 14:34:26 +0000 Subject: [PATCH 15/54] feat: Upgrade Python dependency enterprise-integrated-channels Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/master` --- requirements/common_constraints.txt | 4 ---- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 5 files changed, 4 insertions(+), 8 deletions(-) diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt index af5c9e04c6..368f8fa811 100644 --- a/requirements/common_constraints.txt +++ b/requirements/common_constraints.txt @@ -22,7 +22,3 @@ # elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html # See https://github.com/openedx/edx-platform/issues/35126 for more info elasticsearch<7.14.0 - -# Cause: https://github.com/openedx/edx-lint/issues/458 -# This can be unpinned once https://github.com/openedx/edx-lint/issues/459 has been resolved. - diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 59dfe5c7ec..b07ee46f5d 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -565,7 +565,7 @@ enmerkar==0.7.1 # via enmerkar-underscore enmerkar-underscore==2.4.0 # via -r requirements/edx/kernel.in -enterprise-integrated-channels==0.1.16 +enterprise-integrated-channels==0.1.18 # via -r requirements/edx/bundled.in event-tracking==3.3.0 # via diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 0bebe72b14..808c0f29e1 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -876,7 +876,7 @@ enmerkar-underscore==2.4.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -enterprise-integrated-channels==0.1.16 +enterprise-integrated-channels==0.1.18 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index c70c61054c..489c58c29b 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -654,7 +654,7 @@ enmerkar==0.7.1 # enmerkar-underscore enmerkar-underscore==2.4.0 # via -r requirements/edx/base.txt -enterprise-integrated-channels==0.1.16 +enterprise-integrated-channels==0.1.18 # via -r requirements/edx/base.txt event-tracking==3.3.0 # via diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index fdcf37cd5e..347c166282 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -677,7 +677,7 @@ enmerkar==0.7.1 # enmerkar-underscore enmerkar-underscore==2.4.0 # via -r requirements/edx/base.txt -enterprise-integrated-channels==0.1.16 +enterprise-integrated-channels==0.1.18 # via -r requirements/edx/base.txt event-tracking==3.3.0 # via From 56806007acaf00cd91d8a91e4dfbd8906e7a6997 Mon Sep 17 00:00:00 2001 From: Kyrylo Kholodenko Date: Mon, 6 Oct 2025 13:08:39 +0300 Subject: [PATCH 16/54] refactor: use path instead of re_path --- openedx/features/course_experience/api/v1/urls.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openedx/features/course_experience/api/v1/urls.py b/openedx/features/course_experience/api/v1/urls.py index 30d7c55d29..2c84af437f 100644 --- a/openedx/features/course_experience/api/v1/urls.py +++ b/openedx/features/course_experience/api/v1/urls.py @@ -4,7 +4,7 @@ Contains URLs for the Course Experience API from django.conf import settings -from django.urls import re_path +from django.urls import re_path, path from openedx.features.course_experience.api.v1.views import ( reset_course_deadlines, @@ -21,8 +21,8 @@ urlpatterns += [ reset_course_deadlines, name='course-experience-reset-course-deadlines' ), - re_path( - r'v1/reset_all_course_deadlines/', + path( + 'v1/reset_all_course_deadlines/', reset_all_course_deadlines, name='course-experience-reset-all-course-deadlines', ) From 8b6a94bc8deb636e1050be5f31c009522be3dc62 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Mon, 6 Oct 2025 11:33:25 -0400 Subject: [PATCH 17/54] fix: Only update downstream_customized for upstream-linked blocks (#37412) We only need to track field customizations for upstream-linked (i.e., library-linked) blocks. Thd downstream_customized field is irrelevant for other blocks. It would just add a ton of noise to the OLX. Additionally, we now clear downstream_customized when severing an upstream link. Fixes: https://github.com/openedx/edx-platform/issues/37411 --- cms/lib/xblock/test/test_upstream_sync.py | 19 +++++++++++++++---- cms/lib/xblock/upstream_sync.py | 5 +++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/cms/lib/xblock/test/test_upstream_sync.py b/cms/lib/xblock/test/test_upstream_sync.py index c65c0fcb20..94ee69e5f1 100644 --- a/cms/lib/xblock/test/test_upstream_sync.py +++ b/cms/lib/xblock/test/test_upstream_sync.py @@ -533,15 +533,19 @@ class UpstreamTestCase(ModuleStoreTestCase): """ Does sever_upstream_link correctly disconnect a block from its upstream? """ - # Start with a course block that is linked+synced to a content library block. + # Start with a course block that is linked+synced to a content library block + # and has a customizred title. downstream = BlockFactory.create(category='html', parent=self.unit, upstream=str(self.upstream_key)) sync_from_upstream_block(downstream, self.user) + downstream.display_name = "Downstream Title" + save_xblock_with_callback(downstream, self.user) # (sanity checks) assert downstream.upstream == str(self.upstream_key) assert downstream.upstream_version == 2 assert downstream.upstream_display_name == "Upstream Title V2" - assert downstream.display_name == "Upstream Title V2" + assert downstream.display_name == "Downstream Title" + assert downstream.downstream_customized == ["display_name"] assert downstream.data == "Upstream content V2" assert downstream.copied_from_block is None @@ -552,14 +556,21 @@ class UpstreamTestCase(ModuleStoreTestCase): assert downstream.upstream is None assert downstream.upstream_version is None assert downstream.upstream_display_name is None + assert downstream.downstream_customized == [] - # BUT, the content which was synced into the upstream remains. - assert downstream.display_name == "Upstream Title V2" + # BUT, the content remains. + assert downstream.display_name == "Downstream Title" assert downstream.data == "Upstream content V2" # AND, we have recorded the old upstream as our copied_from_block. assert downstream.copied_from_block == str(self.upstream_key) + # Finally... unlike an upstream-linked block, our unlinked block should not + # have its downstream_customized updated when the title changes. + downstream.display_name = "Downstream Title II" + save_xblock_with_callback(downstream, self.user) + assert downstream.downstream_customized == [] + def test_sync_library_block_tags(self): upstream_lib_block_key = libs.create_library_block(self.library.key, "html", "upstream").usage_key upstream_lib_block = xblock.load_block(upstream_lib_block_key, self.user) diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index ca23f9eb5e..da9a60b422 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -386,6 +386,7 @@ def sever_upstream_link(downstream: XBlock) -> list[XBlock]: downstream.copied_from_block = downstream.upstream downstream.upstream = None downstream.upstream_version = None + downstream.downstream_customized = [] for _, fetched_upstream_field in downstream.get_customizable_fields().items(): # Downstream-only fields don't have an upstream fetch field if fetched_upstream_field is None: @@ -527,6 +528,10 @@ class UpstreamSyncMixin(XBlockMixin): Update `downstream_customized` when a customizable field is modified. """ super().editor_saved(user, old_metadata, old_content) + if not self.upstream: + # If a block does not have an upstream, then we do not need to track its + # customizations. + return customizable_fields = self.get_customizable_fields() new_data = ( self.get_explicitly_set_fields_by_scope(Scope.settings) From 815fd443bb8b52456a190fb5c42a5f5254100142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Mon, 6 Oct 2025 14:30:11 -0300 Subject: [PATCH 18/54] fix: fix fork on multiple migrations (#37422) Fixes a bug where, when running a migration using the fork strategy, it was only looking at the last migration, resulting in a slug reuse, which would cause a component update instead of a new component creation. --- cms/djangoapps/modulestore_migrator/tasks.py | 52 +++++++++++++------ .../modulestore_migrator/tests/test_api.py | 37 +++++++++++++ .../modulestore_migrator/tests/test_tasks.py | 14 ++--- 3 files changed, 80 insertions(+), 23 deletions(-) diff --git a/cms/djangoapps/modulestore_migrator/tasks.py b/cms/djangoapps/modulestore_migrator/tasks.py index 469bf48a65..0c85209887 100644 --- a/cms/djangoapps/modulestore_migrator/tasks.py +++ b/cms/djangoapps/modulestore_migrator/tasks.py @@ -9,6 +9,7 @@ import typing as t from dataclasses import dataclass from datetime import datetime, timezone from enum import Enum +from itertools import groupby from celery import shared_task from celery.utils.log import get_task_logger @@ -20,8 +21,11 @@ from lxml.etree import _ElementTree as XmlTree from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey, UsageKey from opaque_keys.edx.locator import ( - CourseLocator, LibraryLocator, - LibraryLocatorV2, LibraryUsageLocatorV2, LibraryContainerLocator + CourseLocator, + LibraryContainerLocator, + LibraryLocator, + LibraryLocatorV2, + LibraryUsageLocatorV2 ) from openedx_learning.api import authoring as authoring_api from openedx_learning.api.authoring_models import ( @@ -30,21 +34,20 @@ from openedx_learning.api.authoring_models import ( ComponentType, LearningPackage, PublishableEntity, - PublishableEntityVersion, + PublishableEntityVersion ) from user_tasks.tasks import UserTask, UserTaskStatus -from openedx.core.djangoapps.content_libraries.api import ContainerType, get_library +from common.djangoapps.split_modulestore_django.models import SplitModulestoreCourseIndex from openedx.core.djangoapps.content_libraries import api as libraries_api +from openedx.core.djangoapps.content_libraries.api import ContainerType, get_library from openedx.core.djangoapps.content_staging import api as staging_api from xmodule.modulestore import exceptions as modulestore_exceptions from xmodule.modulestore.django import modulestore -from common.djangoapps.split_modulestore_django.models import SplitModulestoreCourseIndex from .constants import CONTENT_STAGING_PURPOSE_TEMPLATE from .data import CompositionLevel, RepeatHandlingStrategy -from .models import ModulestoreSource, ModulestoreMigration, ModulestoreBlockSource, ModulestoreBlockMigration - +from .models import ModulestoreBlockMigration, ModulestoreBlockSource, ModulestoreMigration, ModulestoreSource log = get_task_logger(__name__) @@ -89,7 +92,7 @@ class _MigrationContext: Context for the migration process. """ existing_source_to_target_keys: dict[ # Note: It's intended to be mutable to reflect changes during migration. - UsageKey, PublishableEntity + UsageKey, list[PublishableEntity] ] target_package_id: int target_library_key: LibraryLocatorV2 @@ -105,16 +108,30 @@ class _MigrationContext: return source_key in self.existing_source_to_target_keys def get_existing_target(self, source_key: UsageKey) -> PublishableEntity: - return self.existing_source_to_target_keys[source_key] + """ + Get the target entity for a given source key. + + If the source key is already migrated, return the FIRST target entity. + If the source key is not found, raise a KeyError. + """ + if source_key not in self.existing_source_to_target_keys: + raise KeyError(f"Source key {source_key} not found in existing source to target keys") + + # NOTE: This is a list of PublishableEntities, but we always return the first one. + return self.existing_source_to_target_keys[source_key][0] def add_migration(self, source_key: UsageKey, target: PublishableEntity) -> None: """Update the context with a new migration (keeps it current)""" - self.existing_source_to_target_keys[source_key] = target + if source_key not in self.existing_source_to_target_keys: + self.existing_source_to_target_keys[source_key] = [target] + else: + self.existing_source_to_target_keys[source_key].append(target) def get_existing_target_entity_keys(self, base_key: str) -> set[str]: return set( - publishable_entity.key for _, publishable_entity in - self.existing_source_to_target_keys.items() + publishable_entity.key + for publishable_entity_list in self.existing_source_to_target_keys.values() + for publishable_entity in publishable_entity_list if publishable_entity.key.startswith(base_key) ) @@ -285,10 +302,13 @@ def migrate_from_modulestore( # a given LearningPackage. # We use this mapping to ensure that we don't create duplicate # PublishableEntities during the migration process for a given LearningPackage. + existing_source_to_target_keys: dict[UsageKey, list[PublishableEntity]] = {} + modulestore_blocks = ( + ModulestoreBlockMigration.objects.filter(overall_migration__target=migration.target.id).order_by("source__key") + ) existing_source_to_target_keys = { - block.source.key: block.target for block in ModulestoreBlockMigration.objects.filter( - overall_migration__target=migration.target.id - ) + source_key: list(block.target for block in group) for source_key, group in groupby( + modulestore_blocks, key=lambda x: x.source.key) } migration_context = _MigrationContext( @@ -657,7 +677,7 @@ def _get_distinct_target_usage_key( # Check if we already processed this block and we are not forking. If we are forking, we will # want a new target key. if context.is_already_migrated(source_key) and not context.should_fork_strategy: - log.debug(f"Block {source_key} already exists, reusing existing target") + log.debug(f"Block {source_key} already exists, reusing first existing target") existing_target = context.get_existing_target(source_key) block_id = existing_target.component.local_key diff --git a/cms/djangoapps/modulestore_migrator/tests/test_api.py b/cms/djangoapps/modulestore_migrator/tests/test_api.py index c22df2fc53..13e7e7685d 100644 --- a/cms/djangoapps/modulestore_migrator/tests/test_api.py +++ b/cms/djangoapps/modulestore_migrator/tests/test_api.py @@ -276,6 +276,43 @@ class TestModulestoreMigratorAPI(LibraryTestCase): ) assert second_component.display_name == "Updated Block" + # Update the block again, changing its name + library_block.display_name = "Updated Block Again" + self.store.update_item(library_block, user.id) + + # Migrate again using the Fork strategy + api.start_migration_to_library( + user=user, + source_key=source.key, + target_library_key=self.library_v2.library_key, + composition_level=CompositionLevel.Component.value, + repeat_handling_strategy=RepeatHandlingStrategy.Fork.value, + preserve_url_slugs=True, + forward_source_to_target=False, + ) + + modulestoremigration = ModulestoreMigration.objects.last() + assert modulestoremigration is not None + assert modulestoremigration.repeat_handling_strategy == RepeatHandlingStrategy.Fork.value + + migrated_components_fork = lib_api.get_library_components(self.library_v2.library_key) + assert len(migrated_components_fork) == 3 + + first_component = lib_api.LibraryXBlockMetadata.from_component( + self.library_v2.library_key, migrated_components_fork[0] + ) + assert first_component.display_name == "Original Block" + + second_component = lib_api.LibraryXBlockMetadata.from_component( + self.library_v2.library_key, migrated_components_fork[1] + ) + assert second_component.display_name == "Updated Block" + + third_component = lib_api.LibraryXBlockMetadata.from_component( + self.library_v2.library_key, migrated_components_fork[2] + ) + assert third_component.display_name == "Updated Block Again" + def test_get_migration_info(self): """ Test that the API can retrieve migration info. diff --git a/cms/djangoapps/modulestore_migrator/tests/test_tasks.py b/cms/djangoapps/modulestore_migrator/tests/test_tasks.py index b1c5890e11..309877ae0d 100644 --- a/cms/djangoapps/modulestore_migrator/tests/test_tasks.py +++ b/cms/djangoapps/modulestore_migrator/tests/test_tasks.py @@ -447,7 +447,7 @@ class TestMigrateFromModulestore(ModuleStoreTestCase): title="test_problem" ) - context.existing_source_to_target_keys[source_key] = first_result.entity + context.existing_source_to_target_keys[source_key] = [first_result.entity] second_result = _migrate_component( context=context, @@ -489,7 +489,7 @@ class TestMigrateFromModulestore(ModuleStoreTestCase): title="test_problem" ) - context.existing_source_to_target_keys[source_key_1] = first_result.entity + context.existing_source_to_target_keys[source_key_1] = [first_result.entity] second_result = _migrate_component( context=context, @@ -527,7 +527,7 @@ class TestMigrateFromModulestore(ModuleStoreTestCase): title="original" ) - context.existing_source_to_target_keys[source_key] = first_result.entity + context.existing_source_to_target_keys[source_key] = [first_result.entity] updated_olx = '' second_result = _migrate_component( @@ -708,7 +708,7 @@ class TestMigrateFromModulestore(ModuleStoreTestCase): title="test_problem" ) - context.existing_source_to_target_keys[source_key] = first_result.entity + context.existing_source_to_target_keys[source_key] = [first_result.entity] second_result = _migrate_component( context=context, @@ -863,7 +863,7 @@ class TestMigrateFromModulestore(ModuleStoreTestCase): children=[], ) - context.existing_source_to_target_keys[source_key] = first_result.entity + context.existing_source_to_target_keys[source_key] = [first_result.entity] second_result = _migrate_container( context=context, @@ -909,7 +909,7 @@ class TestMigrateFromModulestore(ModuleStoreTestCase): children=[], ) - context.existing_source_to_target_keys[source_key_1] = first_result.entity + context.existing_source_to_target_keys[source_key_1] = [first_result.entity] second_result = _migrate_container( context=context, @@ -969,7 +969,7 @@ class TestMigrateFromModulestore(ModuleStoreTestCase): children=[], ) - context.existing_source_to_target_keys[source_key] = first_result.entity + context.existing_source_to_target_keys[source_key] = [first_result.entity] second_result = _migrate_container( context=context, From 913598076c1712def426d616b7f3af137f1811ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Mon, 6 Oct 2025 15:53:58 -0300 Subject: [PATCH 19/54] fix: add default to `target_collection_slug` (#37391) adds a default `None` value to the `target_collection_slug` parameter of the migration rest API endpoint, to prevent a `KeyError`. --- cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py b/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py index beb72729cd..df981e8961 100644 --- a/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py +++ b/cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py @@ -46,6 +46,7 @@ class ModulestoreMigrationSerializer(serializers.ModelSerializer): help_text="The target collection slug within the library to import into. Optional.", required=False, allow_blank=True, + default=None, ) forward_source_to_target = serializers.BooleanField( help_text="Forward references of this block source over to the target of this block migration.", From 0c9997ce9237fd20a685b308adb4f660d9ff6af5 Mon Sep 17 00:00:00 2001 From: Rodrigo Mendez Date: Wed, 1 Oct 2025 12:47:33 -0600 Subject: [PATCH 20/54] feat: Implementation of library v2 backup endpoints --- .../content_libraries/api/libraries.py | 48 +++++-- .../content_libraries/rest_api/libraries.py | 127 ++++++++++++++++-- .../content_libraries/rest_api/serializers.py | 35 +++-- .../djangoapps/content_libraries/tasks.py | 96 +++++++++++-- .../content_libraries/tests/base.py | 13 ++ .../content_libraries/tests/test_api.py | 94 +++++++++++++ .../tests/test_content_libraries.py | 34 +++++ .../content_libraries/tests/test_tasks.py | 45 +++++++ .../core/djangoapps/content_libraries/urls.py | 2 + 9 files changed, 446 insertions(+), 48 deletions(-) create mode 100644 openedx/core/djangoapps/content_libraries/tests/test_tasks.py diff --git a/openedx/core/djangoapps/content_libraries/api/libraries.py b/openedx/core/djangoapps/content_libraries/api/libraries.py index ff90c69725..658c55a0e4 100644 --- a/openedx/core/djangoapps/content_libraries/api/libraries.py +++ b/openedx/core/djangoapps/content_libraries/api/libraries.py @@ -41,9 +41,10 @@ could be promoted to the core XBlock API and made generic. """ from __future__ import annotations -from dataclasses import dataclass, field as dataclass_field -from datetime import datetime import logging +from dataclasses import dataclass +from dataclasses import field as dataclass_field +from datetime import datetime from django.conf import settings from django.contrib.auth.models import AbstractUser, AnonymousUser, Group @@ -53,29 +54,24 @@ from django.db import IntegrityError, transaction from django.db.models import Q, QuerySet from django.utils.translation import gettext as _ from opaque_keys.edx.locator import LibraryLocatorV2, LibraryUsageLocatorV2 -from openedx_events.content_authoring.data import ( - ContentLibraryData, -) +from openedx_events.content_authoring.data import ContentLibraryData from openedx_events.content_authoring.signals import ( CONTENT_LIBRARY_CREATED, CONTENT_LIBRARY_DELETED, - CONTENT_LIBRARY_UPDATED, + CONTENT_LIBRARY_UPDATED ) from openedx_learning.api import authoring as authoring_api from openedx_learning.api.authoring_models import Component from organizations.models import Organization +from user_tasks.models import UserTaskArtifact, UserTaskStatus from xblock.core import XBlock from openedx.core.types import User as UserType -from .. import permissions +from .. import permissions, tasks from ..constants import ALL_RIGHTS_RESERVED from ..models import ContentLibrary, ContentLibraryPermission -from .. import tasks -from .exceptions import ( - LibraryAlreadyExists, - LibraryPermissionIntegrityError, -) +from .exceptions import LibraryAlreadyExists, LibraryPermissionIntegrityError log = logging.getLogger(__name__) @@ -105,6 +101,7 @@ __all__ = [ "get_allowed_block_types", "publish_changes", "revert_changes", + "get_backup_task_status", ] @@ -692,3 +689,30 @@ def revert_changes(library_key: LibraryLocatorV2, user_id: int | None = None) -> # Call the event handlers as needed. tasks.wait_for_post_revert_events(draft_change_log, library_key) + + +def get_backup_task_status( + user_id: int, + task_id: str +) -> dict | None: + """ + Get the status of a library backup task. + + Returns a dictionary with the following keys: + - state: One of "Pending", "Exporting", "Succeeded", "Failed" + - url: If state is "Succeeded", the URL where the exported .zip file can be downloaded. Otherwise, None. + If no task is found, returns None. + """ + + try: + task_status = UserTaskStatus.objects.get(task_id=task_id, user_id=user_id) + except UserTaskStatus.DoesNotExist: + return None + + result = {'state': task_status.state, 'url': None} + + if task_status.state == UserTaskStatus.SUCCEEDED: + artifact = UserTaskArtifact.objects.get(status=task_status, name='Output') + result['url'] = artifact.file.storage.url(artifact.file.name) + + return result diff --git a/openedx/core/djangoapps/content_libraries/rest_api/libraries.py b/openedx/core/djangoapps/content_libraries/rest_api/libraries.py index 869b65a3ea..1acdf7bb11 100644 --- a/openedx/core/djangoapps/content_libraries/rest_api/libraries.py +++ b/openedx/core/djangoapps/content_libraries/rest_api/libraries.py @@ -66,6 +66,7 @@ import itertools import json import logging +import edx_api_doc_tools as apidocs from django.conf import settings from django.contrib.auth import authenticate, get_user_model, login from django.contrib.auth.models import Group @@ -78,14 +79,12 @@ from django.views.decorators.clickjacking import xframe_options_exempt from django.views.decorators.csrf import csrf_exempt from django.views.generic.base import TemplateResponseMixin, View from drf_yasg.utils import swagger_auto_schema -from pylti1p3.contrib.django import DjangoCacheDataStorage, DjangoDbToolConf, DjangoMessageLaunch, DjangoOIDCLogin -from pylti1p3.exception import LtiException, OIDCException - -import edx_api_doc_tools as apidocs from opaque_keys.edx.locator import LibraryLocatorV2, LibraryUsageLocatorV2 from organizations.api import ensure_organization from organizations.exceptions import InvalidOrganizationException from organizations.models import Organization +from pylti1p3.contrib.django import DjangoCacheDataStorage, DjangoDbToolConf, DjangoMessageLaunch, DjangoOIDCLogin +from pylti1p3.exception import LtiException, OIDCException from rest_framework import status from rest_framework.exceptions import NotFound, PermissionDenied, ValidationError from rest_framework.generics import GenericAPIView @@ -93,12 +92,15 @@ from rest_framework.response import Response from rest_framework.views import APIView from rest_framework.viewsets import GenericViewSet +import openedx.core.djangoapps.site_configuration.helpers as configuration_helpers from cms.djangoapps.contentstore.views.course import ( get_allowed_organizations_for_libraries, - user_can_create_organizations, + user_can_create_organizations ) from openedx.core.djangoapps.content_libraries import api, permissions +from openedx.core.djangoapps.content_libraries.api.libraries import get_backup_task_status from openedx.core.djangoapps.content_libraries.rest_api.serializers import ( + ContentLibraryAddPermissionByEmailSerializer, ContentLibraryBlockImportTaskCreateSerializer, ContentLibraryBlockImportTaskSerializer, ContentLibraryFilterSerializer, @@ -106,20 +108,20 @@ from openedx.core.djangoapps.content_libraries.rest_api.serializers import ( ContentLibraryPermissionLevelSerializer, ContentLibraryPermissionSerializer, ContentLibraryUpdateSerializer, + LibraryBackupResponseSerializer, + LibraryBackupTaskStatusSerializer, LibraryXBlockCreationSerializer, LibraryXBlockMetadataSerializer, LibraryXBlockTypeSerializer, - ContentLibraryAddPermissionByEmailSerializer, - PublishableItemSerializer, + PublishableItemSerializer ) -import openedx.core.djangoapps.site_configuration.helpers as configuration_helpers -from openedx.core.lib.api.view_utils import view_auth_classes +from openedx.core.djangoapps.content_libraries.tasks import backup_library from openedx.core.djangoapps.safe_sessions.middleware import mark_user_change_as_expected from openedx.core.djangoapps.xblock import api as xblock_api +from openedx.core.lib.api.view_utils import view_auth_classes -from .utils import convert_exceptions from ..models import ContentLibrary, LtiGradedResource, LtiProfile - +from .utils import convert_exceptions User = get_user_model() log = logging.getLogger(__name__) @@ -685,6 +687,109 @@ class LibraryImportTaskViewSet(GenericViewSet): return Response(ContentLibraryBlockImportTaskSerializer(import_task).data) +# Library Backup Views +# ==================== + +@method_decorator(non_atomic_requests, name="dispatch") +@view_auth_classes() +class LibraryBackupView(APIView): + """ + **Use Case** + * Start an asynchronous task to back up the content of a library to a .zip file + * Get a status on an asynchronous export task + + **Example Requests** + POST /api/libraries/v2/{library_id}/backup/ + GET /api/libraries/v2/{library_id}/backup/?task_id={task_id} + + **POST Response Values** + + If the import task is started successfully, an HTTP 200 "OK" response is + returned. + + The HTTP 200 response has the following values: + + * task_id: UUID of the created task, usable for checking status + + **Example POST Response** + + { + "task_id": "7069b95b-ccea-4214-b6db-e00f27065bf7" + } + + **GET Parameters** + + A GET request must include the following parameters: + + * task_id: (required) The UUID of the task to check. + + **GET Response Values** + + If the import task is found successfully by the UUID provided, an HTTP + 200 "OK" response is returned. + + The HTTP 200 response has the following values: + + * state: String description of the state of the task. + Possible states: "Pending", "Exporting", "Succeeded", "Failed". + * url: (may be null) If the task is complete, a URL to download the .zip file + + **Example GET Response** + { + "state": "Succeeded", + "url": "/media/user_tasks/2025/10/03/lib-wgu-csprob-2025-10-03-153633.zip" + } + + """ + + @apidocs.schema( + body=None, + responses={200: LibraryBackupResponseSerializer} + ) + @convert_exceptions + def post(self, request, lib_key_str): + """ + Start backup task for the specified library. + """ + library_key = LibraryLocatorV2.from_string(lib_key_str) + # Using CAN_EDIT_THIS_CONTENT_LIBRARY permission for now. This should eventually become its own permission + api.require_permission_for_library_key(library_key, request.user, permissions.CAN_EDIT_THIS_CONTENT_LIBRARY) + + async_result = backup_library.delay(request.user.id, str(library_key)) + result = {'task_id': async_result.task_id} + + return Response(LibraryBackupResponseSerializer(result).data) + + @apidocs.schema( + parameters=[ + apidocs.query_parameter( + 'task_id', + str, + description="The ID of the backup task to retrieve." + ), + ], + responses={200: LibraryBackupTaskStatusSerializer} + ) + @convert_exceptions + def get(self, request, lib_key_str): + """ + Get the status of the specified backup task for the specified library. + """ + library_key = LibraryLocatorV2.from_string(lib_key_str) + # Using CAN_EDIT_THIS_CONTENT_LIBRARY permission for now. This should eventually become its own permission + api.require_permission_for_library_key(library_key, request.user, permissions.CAN_EDIT_THIS_CONTENT_LIBRARY) + + task_id = request.query_params.get('task_id', None) + if not task_id: + raise ValidationError(detail={'task_id': _('This field is required.')}) + result = get_backup_task_status(request.user.id, task_id) + + if not result: + raise NotFound(detail="No backup found for this library.") + + return Response(LibraryBackupTaskStatusSerializer(result).data) + + # LTI 1.3 Views # ============= diff --git a/openedx/core/djangoapps/content_libraries/rest_api/serializers.py b/openedx/core/djangoapps/content_libraries/rest_api/serializers.py index 9cdbe43901..3b4dba09a1 100644 --- a/openedx/core/djangoapps/content_libraries/rest_api/serializers.py +++ b/openedx/core/djangoapps/content_libraries/rest_api/serializers.py @@ -3,26 +3,22 @@ Serializers for the content libraries REST API """ # pylint: disable=abstract-method from django.core.validators import validate_unicode_slug +from opaque_keys import InvalidKeyError, OpaqueKey +from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocatorV2 +from openedx_learning.api.authoring_models import Collection from rest_framework import serializers from rest_framework.exceptions import ValidationError -from opaque_keys import OpaqueKey -from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocatorV2 -from opaque_keys import InvalidKeyError - -from openedx_learning.api.authoring_models import Collection from openedx.core.djangoapps.content_libraries.api.containers import ContainerType -from openedx.core.djangoapps.content_libraries.constants import ( - ALL_RIGHTS_RESERVED, - LICENSE_OPTIONS, -) +from openedx.core.djangoapps.content_libraries.constants import ALL_RIGHTS_RESERVED, LICENSE_OPTIONS from openedx.core.djangoapps.content_libraries.models import ( - ContentLibraryPermission, ContentLibraryBlockImportTask, - ContentLibrary + ContentLibrary, + ContentLibraryBlockImportTask, + ContentLibraryPermission ) from openedx.core.lib.api.serializers import CourseKeyField -from .. import permissions +from .. import permissions DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ' @@ -416,3 +412,18 @@ class ContainerHierarchySerializer(serializers.Serializer): units = serializers.ListField(child=ContainerHierarchyMemberSerializer(), allow_empty=True) components = serializers.ListField(child=ContainerHierarchyMemberSerializer(), allow_empty=True) object_key = OpaqueKeySerializer() + + +class LibraryBackupResponseSerializer(serializers.Serializer): + """ + Serializer for the response after requesting a backup of a content library. + """ + task_id = serializers.CharField() + + +class LibraryBackupTaskStatusSerializer(serializers.Serializer): + """ + Serializer for checking the status of a library backup task. + """ + state = serializers.CharField() + url = serializers.URLField(allow_null=True) diff --git a/openedx/core/djangoapps/content_libraries/tasks.py b/openedx/core/djangoapps/content_libraries/tasks.py index ebc8e27830..8c362dd526 100644 --- a/openedx/core/djangoapps/content_libraries/tasks.py +++ b/openedx/core/djangoapps/content_libraries/tasks.py @@ -17,37 +17,44 @@ Architecture note: from __future__ import annotations import logging +import os +from datetime import datetime +from tempfile import mkdtemp from celery import shared_task -from celery_utils.logged_task import LoggedTask from celery.utils.log import get_task_logger -from edx_django_utils.monitoring import set_code_owner_attribute, set_code_owner_attribute_from_module +from celery_utils.logged_task import LoggedTask +from django.core.files import File +from django.utils.text import slugify +from edx_django_utils.monitoring import ( + set_code_owner_attribute, + set_code_owner_attribute_from_module, + set_custom_attribute +) from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import ( BlockUsageLocator, LibraryCollectionLocator, LibraryContainerLocator, - LibraryLocatorV2, -) -from openedx_learning.api import authoring as authoring_api -from openedx_learning.api.authoring_models import DraftChangeLog, PublishLog -from openedx_events.content_authoring.data import ( - LibraryBlockData, - LibraryCollectionData, - LibraryContainerData, + LibraryLocatorV2 ) +from openedx_events.content_authoring.data import LibraryBlockData, LibraryCollectionData, LibraryContainerData from openedx_events.content_authoring.signals import ( LIBRARY_BLOCK_CREATED, LIBRARY_BLOCK_DELETED, - LIBRARY_BLOCK_UPDATED, LIBRARY_BLOCK_PUBLISHED, + LIBRARY_BLOCK_UPDATED, LIBRARY_COLLECTION_UPDATED, LIBRARY_CONTAINER_CREATED, LIBRARY_CONTAINER_DELETED, - LIBRARY_CONTAINER_UPDATED, LIBRARY_CONTAINER_PUBLISHED, + LIBRARY_CONTAINER_UPDATED ) - +from openedx_learning.api import authoring as authoring_api +from openedx_learning.api.authoring import create_zip_file as create_lib_zip_file +from openedx_learning.api.authoring_models import DraftChangeLog, PublishLog +from path import Path +from user_tasks.models import UserTaskArtifact from user_tasks.tasks import UserTask, UserTaskStatus from xblock.fields import Scope @@ -477,3 +484,66 @@ def _copy_overrides( dest_block=store.get_item(dest_child_key), ) store.update_item(dest_block, user_id) + + +class LibraryBackupTask(UserTask): # pylint: disable=abstract-method + """ + Base class for tasks related with Library backup functionality. + """ + + @classmethod + def generate_name(cls, arguments_dict) -> str: + """ + Create a name for this particular backup task instance. + + Should be both: + a. semi human-friendly + b. something we can query in order to determine whether the library has a task in progress + + Arguments: + arguments_dict (dict): The arguments given to the task function + + Returns: + str: The generated name + """ + key = arguments_dict['library_key_str'] + return f'Backup of {key}' + + +@shared_task(base=LibraryBackupTask, bind=True) +# Note: The decorator @set_code_owner_attribute cannot be used here because the UserTaskMixin +# does stack inspection and can't handle additional decorators. +def backup_library(self, user_id: int, library_key_str: str) -> None: + """ + Export a library to a .zip archive and prepare it for download. + Possible Task states: + - Pending: Task is created but not started yet. + - Exporting: Task is running and the library is being exported. + - Succeeded: Task completed successfully and the exported file is available for download. + - Failed: Task failed and the export did not complete. + """ + ensure_cms("backup_library may only be executed in a CMS context") + set_code_owner_attribute_from_module(__name__) + library_key = LibraryLocatorV2.from_string(library_key_str) + + try: + self.status.set_state('Exporting') + set_custom_attribute("exporting_started", str(library_key)) + + root_dir = Path(mkdtemp()) + sanitized_lib_key = str(library_key).replace(":", "-") + sanitized_lib_key = slugify(sanitized_lib_key, allow_unicode=True) + timestamp = datetime.now().strftime("%Y-%m-%d-%H%M%S") + filename = f'{sanitized_lib_key}-{timestamp}.zip' + file_path = os.path.join(root_dir, filename) + create_lib_zip_file(lp_key=str(library_key), path=file_path) + set_custom_attribute("exporting_completed", str(library_key)) + + with open(file_path, 'rb') as zipfile: + artifact = UserTaskArtifact(status=self.status, name='Output') + artifact.file.save(name=os.path.basename(zipfile.name), content=File(zipfile)) + artifact.save() + except Exception as exception: # pylint: disable=broad-except + TASK_LOGGER.exception('Error exporting library %s', library_key, exc_info=True) + if self.status.state != UserTaskStatus.FAILED: + self.status.fail({'raw_error_msg': str(exception)}) diff --git a/openedx/core/djangoapps/content_libraries/tests/base.py b/openedx/core/djangoapps/content_libraries/tests/base.py index 1c1bf1b137..7002f41eca 100644 --- a/openedx/core/djangoapps/content_libraries/tests/base.py +++ b/openedx/core/djangoapps/content_libraries/tests/base.py @@ -32,6 +32,8 @@ URL_LIB_TEAM = URL_LIB_DETAIL + 'team/' # Get the list of users/groups authoriz URL_LIB_TEAM_USER = URL_LIB_TEAM + 'user/{username}/' # Add/edit/remove a user's permission to use this library URL_LIB_TEAM_GROUP = URL_LIB_TEAM + 'group/{group_name}/' # Add/edit/remove a group's permission to use this library URL_LIB_PASTE_CLIPBOARD = URL_LIB_DETAIL + 'paste_clipboard/' # Paste user clipboard (POST) containing Xblock data +URL_LIB_BACKUP = URL_LIB_DETAIL + 'backup/' # Start a backup task for this library +URL_LIB_BACKUP_GET = URL_LIB_BACKUP + '?{query_params}' # Get status on a backup task for this library URL_LIB_BLOCK = URL_PREFIX + 'blocks/{block_key}/' # Get data about a block, or delete it URL_LIB_BLOCK_PUBLISH = URL_LIB_BLOCK + 'publish/' # Publish changes from a specified XBlock URL_LIB_BLOCK_OLX = URL_LIB_BLOCK + 'olx/' # Get or set the OLX of the specified XBlock @@ -319,6 +321,17 @@ class ContentLibrariesRestApiTest(APITransactionTestCase): url = URL_LIB_PASTE_CLIPBOARD.format(lib_key=lib_key) return self._api('post', url, {}, expect_response) + def _start_library_backup_task(self, lib_key, expect_response=200): + """ Start a backup task for this library """ + url = URL_LIB_BACKUP.format(lib_key=lib_key) + return self._api('post', url, {}, expect_response) + + def _get_library_backup_task(self, lib_key, task_id, expect_response=200): + """ Get the status of a backup task for this library """ + query_params = urlencode({"task_id": task_id}) + url = URL_LIB_BACKUP_GET.format(lib_key=lib_key, query_params=query_params) + return self._api('get', url, None, expect_response) + def _render_block_view(self, block_key, view_name, version=None, expect_response=200): """ Render an XBlock's view in the active application's runtime. diff --git a/openedx/core/djangoapps/content_libraries/tests/test_api.py b/openedx/core/djangoapps/content_libraries/tests/test_api.py index 6756e4373a..d92a97530c 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_api.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_api.py @@ -4,9 +4,11 @@ Tests for Content Library internal api. import base64 import hashlib +import uuid from unittest import mock from django.test import TestCase +from user_tasks.models import UserTaskStatus from opaque_keys.edx.keys import ( CourseKey, @@ -1309,3 +1311,95 @@ class ContentLibraryContainersTest(ContentLibrariesRestApiTest): ), }, ) + + +class ContentLibraryExportTest(ContentLibrariesRestApiTest): + """ + Tests for Content Library API export methods. + """ + + def setUp(self) -> None: + super().setUp() + + # Create Content Libraries + self._create_library("test-lib-exp-1", "Test Library Export 1") + + # Fetch the created ContentLibrary objects so we can access their learning_package.id + self.lib1 = ContentLibrary.objects.get(slug="test-lib-exp-1") + self.wrong_task_id = '11111111-1111-1111-1111-111111111111' + + def test_get_backup_task_status_no_task(self) -> None: + status = api.get_backup_task_status(self.user.id, "") + assert status is None + + def test_get_backup_task_status_wrong_task_id(self) -> None: + status = api.get_backup_task_status(self.user.id, task_id=self.wrong_task_id) + assert status is None + + def test_get_backup_task_status_in_progress(self) -> None: + # Create a mock UserTaskStatus in IN_PROGRESS state + task_id = str(uuid.uuid4()) + mock_task = UserTaskStatus( + task_id=task_id, + user_id=self.user.id, + name=f"Export of {self.lib1.library_key}", + state=UserTaskStatus.IN_PROGRESS + ) + + with mock.patch( + 'openedx.core.djangoapps.content_libraries.api.libraries.UserTaskStatus.objects.get' + ) as mock_get: + mock_get.return_value = mock_task + + status = api.get_backup_task_status(self.user.id, task_id=task_id) + assert status is not None + assert status['state'] == UserTaskStatus.IN_PROGRESS + assert status['url'] is None + + def test_get_backup_task_status_succeeded(self) -> None: + # Create a mock UserTaskStatus in SUCCEEDED state + task_id = str(uuid.uuid4()) + mock_task = UserTaskStatus( + task_id=task_id, + user_id=self.user.id, + name=f"Export of {self.lib1.library_key}", + state=UserTaskStatus.SUCCEEDED + ) + + # Create a mock UserTaskArtifact + mock_artifact = mock.Mock() + mock_artifact.file.storage.url.return_value = "/media/user_tasks/2025/10/01/library-libOEXCSPROB_mOw1rPL.zip" + + with mock.patch( + 'openedx.core.djangoapps.content_libraries.api.libraries.UserTaskStatus.objects.get' + ) as mock_get, mock.patch( + 'openedx.core.djangoapps.content_libraries.api.libraries.UserTaskArtifact.objects.get' + ) as mock_artifact_get: + + mock_get.return_value = mock_task + mock_artifact_get.return_value = mock_artifact + + status = api.get_backup_task_status(self.user.id, task_id=task_id) + assert status is not None + assert status['state'] == UserTaskStatus.SUCCEEDED + assert status['url'] == "/media/user_tasks/2025/10/01/library-libOEXCSPROB_mOw1rPL.zip" + + def test_get_backup_task_status_failed(self) -> None: + # Create a mock UserTaskStatus in FAILED state + task_id = str(uuid.uuid4()) + mock_task = UserTaskStatus( + task_id=task_id, + user_id=self.user.id, + name=f"Export of {self.lib1.library_key}", + state=UserTaskStatus.FAILED + ) + + with mock.patch( + 'openedx.core.djangoapps.content_libraries.api.libraries.UserTaskStatus.objects.get' + ) as mock_get: + mock_get.return_value = mock_task + + status = api.get_backup_task_status(self.user.id, task_id=task_id) + assert status is not None + assert status['state'] == UserTaskStatus.FAILED + assert status['url'] is None diff --git a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py index e2fec3aee1..8fcc8b9a68 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py @@ -823,6 +823,40 @@ class ContentLibrariesTestCase(ContentLibrariesRestApiTest): "id": f"lb:CL-TEST:test_lib_paste_clipboard:problem:{pasted_usage_key.block_id}", }) + def test_start_library_backup(self): + """ + Test starting a backup operation on a content library. + """ + author = UserFactory.create(username="Author", email="author@example.com", is_staff=True) + with self.as_user(author): + lib = self._create_library( + slug="test_lib_backup", + title="Backup Test Library", + description="Testing backup for library" + ) + lib_id = lib["id"] + response = self._start_library_backup_task(lib_id) + assert response["task_id"] is not None + + def test_get_library_backup_status(self): + """ + Test getting the status of a backup operation on a content library. + """ + author = UserFactory.create(username="Author", email="author@example.com", is_staff=True) + with self.as_user(author): + lib = self._create_library( + slug="test_lib_backup_status", + title="Backup Status Test Library", + description="Testing backup status for library" + ) + lib_id = lib["id"] + response = self._start_library_backup_task(lib_id) + task_id = response["task_id"] + + # Now check the status of the backup task + status_response = self._get_library_backup_task(lib_id, task_id) + assert status_response["state"] in ["Pending", "Exporting", "Succeeded", "Failed"] + @override_settings(LIBRARY_ENABLED_BLOCKS=['problem', 'video', 'html']) def test_library_get_enabled_blocks(self): expected = [ diff --git a/openedx/core/djangoapps/content_libraries/tests/test_tasks.py b/openedx/core/djangoapps/content_libraries/tests/test_tasks.py new file mode 100644 index 0000000000..4098b2a8ff --- /dev/null +++ b/openedx/core/djangoapps/content_libraries/tests/test_tasks.py @@ -0,0 +1,45 @@ +""" +Unit tests for content libraries Celery tasks +""" + +from ..models import ContentLibrary +from .base import ContentLibrariesRestApiTest + +from openedx.core.djangoapps.content_libraries.tasks import backup_library +from user_tasks.models import UserTaskArtifact + + +class ContentLibraryBackupTaskTest(ContentLibrariesRestApiTest): + """ + Tests for Content Library export task. + """ + + def setUp(self) -> None: + super().setUp() + + # Create Content Libraries + self._create_library("test-lib-task-1", "Test Library Task 1") + + # Fetch the created ContentLibrary objects so we can access their learning_package.id + self.lib1 = ContentLibrary.objects.get(slug="test-lib-task-1") + self.wrong_task_id = '11111111-1111-1111-1111-111111111111' + + def test_backup_task_returns_task_id(self): + result = backup_library.delay(self.user.id, str(self.lib1.library_key)) + assert result.task_id is not None + + def test_backup_task_success(self): + result = backup_library.delay(self.user.id, str(self.lib1.library_key)) + assert result.state == 'SUCCESS' + # Ensure an artifact was created with the output file + artifact = UserTaskArtifact.objects.filter(status__task_id=result.task_id, name='Output').first() + assert artifact is not None + assert artifact.file.name.endswith('.zip') + + def test_backup_task_failure(self): + result = backup_library.delay(self.user.id, self.wrong_task_id) + assert result.state == 'FAILURE' + # Ensure an error artifact was created + artifact = UserTaskArtifact.objects.filter(status__task_id=result.task_id, name='Error').first() + assert artifact is not None + assert artifact.text is not None diff --git a/openedx/core/djangoapps/content_libraries/urls.py b/openedx/core/djangoapps/content_libraries/urls.py index d0e30a4200..f59a36e6f0 100644 --- a/openedx/core/djangoapps/content_libraries/urls.py +++ b/openedx/core/djangoapps/content_libraries/urls.py @@ -54,6 +54,8 @@ urlpatterns = [ path('import_blocks/', include(import_blocks_router.urls)), # Paste contents of clipboard into library path('paste_clipboard/', libraries.LibraryPasteClipboardView.as_view()), + # Start a backup task for this library + path('backup/', libraries.LibraryBackupView.as_view()), # Library Collections path('', include(library_collections_router.urls)), ])), From 242a69d06b9280755c1a8163624fefd85039c510 Mon Sep 17 00:00:00 2001 From: Usama Sadiq Date: Tue, 7 Oct 2025 09:27:24 +0500 Subject: [PATCH 21/54] Fix upgrade job pin cryptography (#37436) * fix: pin cryptography to fix the upgrade job * fix: pin pact-python<3.0.0 --------- Co-authored-by: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> --- requirements/common_constraints.txt | 4 - requirements/constraints.txt | 11 ++ requirements/edx-sandbox/base.txt | 18 +-- requirements/edx/assets.txt | 2 +- requirements/edx/base.txt | 84 +++++++------ requirements/edx/coverage.txt | 6 +- requirements/edx/development.txt | 118 ++++++++--------- requirements/edx/doc.txt | 88 ++++++------- requirements/edx/semgrep.txt | 119 +++++++++++++----- requirements/edx/testing.txt | 104 +++++++-------- requirements/pip-tools.txt | 4 +- .../structures_pruning/requirements/base.txt | 2 +- .../requirements/testing.txt | 2 +- scripts/user_retirement/requirements/base.txt | 34 ++--- .../user_retirement/requirements/testing.txt | 36 +++--- scripts/xblock/requirements.txt | 2 +- 16 files changed, 357 insertions(+), 277 deletions(-) diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt index af5c9e04c6..368f8fa811 100644 --- a/requirements/common_constraints.txt +++ b/requirements/common_constraints.txt @@ -22,7 +22,3 @@ # elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html # See https://github.com/openedx/edx-platform/issues/35126 for more info elasticsearch<7.14.0 - -# Cause: https://github.com/openedx/edx-lint/issues/458 -# This can be unpinned once https://github.com/openedx/edx-lint/issues/459 has been resolved. - diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 3c36ffafcd..f78de74731 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -125,3 +125,14 @@ xmlsec==1.3.14 # https://github.com/django-commons/django-debug-toolbar/issues/2172 # Pin this back to the previous version until that bug is fixed. django-debug-toolbar<6.0.0 + +# Date 2025-10-07 +# Cryptography 46.0.0 conflicts with system dependencies needed for snowflake-connector-python +# snowflake-connector-python comes as a dependency of edx-enterprise so it can not be directly pinned here. +# See issue https://github.com/openedx/edx-platform/issues/37417 for details on this. +# This can be unpinned once snowflake-connector-python==4.0.0 is available (contains the fix). +# pact-python==3.0.0 also removes cffi dependency and is causing the upgrade build to fail +# This should also be removed together with cryptography constraint. +# Issue: https://github.com/openedx/edx-platform/issues/37435 +cryptography<46.0.0 +pact-python<3.0.0 diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index ec9a8ff522..a2013ea748 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -8,17 +8,19 @@ cffi==2.0.0 # via cryptography chem==2.0.0 # via -r requirements/edx-sandbox/base.in -click==8.2.1 +click==8.3.0 # via nltk codejail-includes==2.0.0 # via -r requirements/edx-sandbox/base.in contourpy==1.3.3 # via matplotlib cryptography==45.0.7 - # via -r requirements/edx-sandbox/base.in + # via + # -c requirements/constraints.txt + # -r requirements/edx-sandbox/base.in cycler==0.12.1 # via matplotlib -fonttools==4.59.2 +fonttools==4.60.1 # via matplotlib joblib==1.5.2 # via nltk @@ -30,9 +32,9 @@ lxml[html-clean]==5.3.2 # -r requirements/edx-sandbox/base.in # lxml-html-clean # openedx-calc -lxml-html-clean==0.4.2 +lxml-html-clean==0.4.3 # via lxml -markupsafe==3.0.2 +markupsafe==3.0.3 # via # chem # openedx-calc @@ -42,7 +44,7 @@ mpmath==1.3.0 # via sympy networkx==3.5 # via -r requirements/edx-sandbox/base.in -nltk==3.9.1 +nltk==3.9.2 # via # -r requirements/edx-sandbox/base.in # chem @@ -62,7 +64,7 @@ pillow==11.3.0 # via matplotlib pycparser==2.23 # via cffi -pyparsing==3.2.4 +pyparsing==3.2.5 # via # -r requirements/edx-sandbox/base.in # chem @@ -72,7 +74,7 @@ python-dateutil==2.9.0.post0 # via matplotlib random2==1.0.2 # via -r requirements/edx-sandbox/base.in -regex==2025.9.1 +regex==2025.9.18 # via nltk scipy==1.16.2 # via diff --git a/requirements/edx/assets.txt b/requirements/edx/assets.txt index bb6693f4dc..f66289e09b 100644 --- a/requirements/edx/assets.txt +++ b/requirements/edx/assets.txt @@ -4,7 +4,7 @@ # # make upgrade # -click==8.2.1 +click==8.3.0 # via -r requirements/edx/assets.in libsass==0.10.0 # via diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index d222c2a661..7304601398 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -8,7 +8,7 @@ acid-xblock==0.4.1 # via -r requirements/edx/kernel.in aiohappyeyeballs==2.6.1 # via aiohttp -aiohttp==3.12.15 +aiohttp==3.13.0 # via # geoip2 # openai @@ -22,18 +22,18 @@ aniso8601==10.0.1 # via edx-tincan-py35 annotated-types==0.7.0 # via pydantic -anyio==4.10.0 +anyio==4.11.0 # via httpx appdirs==1.4.4 # via fs -asgiref==3.9.1 +asgiref==3.10.0 # via # django # django-cors-headers # django-countries asn1crypto==1.5.1 # via snowflake-connector-python -attrs==25.3.0 +attrs==25.4.0 # via # -r requirements/edx/kernel.in # aiohttp @@ -50,13 +50,13 @@ babel==2.17.0 # enmerkar-underscore backoff==1.10.0 # via analytics-python -bcrypt==4.3.0 +bcrypt==5.0.0 # via paramiko -beautifulsoup4==4.13.5 +beautifulsoup4==4.14.2 # via # openedx-forum # pynliner -billiard==4.2.1 +billiard==4.2.2 # via celery bleach[css]==6.2.0 # via @@ -68,14 +68,14 @@ bleach[css]==6.2.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/kernel.in -boto3==1.40.31 +boto3==1.40.46 # via # -r requirements/edx/kernel.in # django-ses # fs-s3fs # ora2 # snowflake-connector-python -botocore==1.40.31 +botocore==1.40.46 # via # -r requirements/edx/kernel.in # boto3 @@ -85,7 +85,7 @@ bridgekeeper==0.9 # via -r requirements/edx/kernel.in cachecontrol==0.14.3 # via firebase-admin -cachetools==5.5.2 +cachetools==6.2.0 # via # edxval # google-auth @@ -102,7 +102,7 @@ celery==5.5.3 # enterprise-integrated-channels # event-tracking # openedx-learning -certifi==2025.8.3 +certifi==2025.10.5 # via # elasticsearch # httpcore @@ -122,7 +122,7 @@ charset-normalizer==3.4.3 # snowflake-connector-python chem==2.0.0 # via -r requirements/edx/kernel.in -click==8.2.1 +click==8.3.0 # via # celery # click-didyoumean @@ -147,6 +147,7 @@ crowdsourcehinter-xblock==0.8 # via -r requirements/edx/bundled.in cryptography==45.0.7 # via + # -c requirements/constraints.txt # -r requirements/edx/kernel.in # django-fernet-fields-v2 # edx-enterprise @@ -257,7 +258,7 @@ django-config-models==2.9.0 # edx-name-affirmation # enterprise-integrated-channels # lti-consumer-xblock -django-cors-headers==4.8.0 +django-cors-headers==4.9.0 # via -r requirements/edx/kernel.in django-countries==7.6.1 # via @@ -315,7 +316,7 @@ django-mptt==0.18.0 # openedx-django-wiki django-multi-email-field==0.8.0 # via edx-enterprise -django-mysql==4.18.0 +django-mysql==4.19.0 # via -r requirements/edx/kernel.in django-oauth-toolkit==1.7.1 # via @@ -403,7 +404,7 @@ drf-jwt==1.19.2 # via edx-drf-extensions drf-spectacular==0.28.0 # via -r requirements/edx/kernel.in -drf-yasg==1.21.10 +drf-yasg==1.21.11 # via # django-user-tasks # edx-api-doc-tools @@ -413,7 +414,7 @@ edx-api-doc-tools==2.1.0 # via # -r requirements/edx/kernel.in # edx-name-affirmation -edx-auth-backends==4.6.0 +edx-auth-backends==4.6.1 # via -r requirements/edx/kernel.in edx-bulk-grades==1.2.0 # via @@ -440,7 +441,7 @@ edx-django-release-util==1.5.0 # edxval edx-django-sites-extensions==5.1.0 # via -r requirements/edx/kernel.in -edx-django-utils==8.0.0 +edx-django-utils==8.0.1 # via # -r requirements/edx/kernel.in # django-config-models @@ -527,7 +528,7 @@ edx-search==4.3.0 # openedx-forum edx-sga==0.26.0 # via -r requirements/edx/bundled.in -edx-submissions==3.11.1 +edx-submissions==3.12.0 # via # -r requirements/edx/kernel.in # ora2 @@ -564,7 +565,7 @@ enmerkar==0.7.1 # via enmerkar-underscore enmerkar-underscore==2.4.0 # via -r requirements/edx/kernel.in -enterprise-integrated-channels==0.1.16 +enterprise-integrated-channels==0.1.18 # via -r requirements/edx/bundled.in event-tracking==3.3.0 # via @@ -578,7 +579,7 @@ filelock==3.19.1 # via snowflake-connector-python firebase-admin==7.1.0 # via edx-ace -frozenlist==1.7.0 +frozenlist==1.8.0 # via # aiohttp # aiosignal @@ -596,13 +597,13 @@ geoip2==5.1.0 # via -r requirements/edx/kernel.in glob2==0.7 # via -r requirements/edx/kernel.in -google-api-core[grpc]==2.25.1 +google-api-core[grpc]==2.25.2 # via # firebase-admin # google-cloud-core # google-cloud-firestore # google-cloud-storage -google-auth==2.40.3 +google-auth==2.41.1 # via # google-api-core # google-cloud-core @@ -626,11 +627,11 @@ googleapis-common-protos==1.70.0 # via # google-api-core # grpcio-status -grpcio==1.74.0 +grpcio==1.75.1 # via # google-api-core # grpcio-status -grpcio-status==1.74.0 +grpcio-status==1.75.1 # via google-api-core gunicorn==23.0.0 # via -r requirements/edx/kernel.in @@ -732,7 +733,7 @@ lxml[html-clean]==5.3.2 # python3-saml # xblock # xmlsec -lxml-html-clean==0.4.2 +lxml-html-clean==0.4.3 # via lxml mailsnake==1.6.4 # via -r requirements/edx/bundled.in @@ -749,7 +750,7 @@ markdown==3.9 # openedx-django-wiki # staff-graded-xblock # xblock-poll -markupsafe==3.0.2 +markupsafe==3.0.3 # via # chem # jinja2 @@ -772,7 +773,7 @@ mpmath==1.3.0 # via sympy msgpack==1.1.1 # via cachecontrol -multidict==6.6.4 +multidict==6.7.0 # via # aiohttp # yarl @@ -784,7 +785,7 @@ nh3==0.3.0 # via # -r requirements/edx/kernel.in # xblocks-contrib -nltk==3.9.1 +nltk==3.9.2 # via chem nodeenv==1.9.1 # via -r requirements/edx/kernel.in @@ -884,7 +885,7 @@ polib==1.2.0 # via edx-i18n-tools prompt-toolkit==3.0.52 # via click-repl -propcache==0.3.2 +propcache==0.4.0 # via # aiohttp # yarl @@ -899,7 +900,7 @@ protobuf==6.32.1 # googleapis-common-protos # grpcio-status # proto-plus -psutil==7.0.0 +psutil==7.1.0 # via # -r requirements/edx/kernel.in # edx-django-utils @@ -919,7 +920,7 @@ pycryptodomex==3.23.0 # -r requirements/edx/kernel.in # edx-proctoring # lti-consumer-xblock -pydantic==2.11.9 +pydantic==2.11.10 # via camel-converter pydantic-core==2.33.2 # via pydantic @@ -956,9 +957,9 @@ pynacl==1.6.0 # paramiko pynliner==0.8.0 # via -r requirements/edx/kernel.in -pyopenssl==25.2.0 +pyopenssl==25.3.0 # via snowflake-connector-python -pyparsing==3.2.4 +pyparsing==3.2.5 # via # chem # openedx-calc @@ -1010,7 +1011,7 @@ pytz==2025.2 # xblock pyuca==1.2 # via -r requirements/edx/kernel.in -pyyaml==6.0.2 +pyyaml==6.0.3 # via # -r requirements/edx/kernel.in # code-annotations @@ -1032,7 +1033,7 @@ referencing==0.36.2 # via # jsonschema # jsonschema-specifications -regex==2025.9.1 +regex==2025.9.18 # via nltk requests==2.32.5 # via @@ -1084,9 +1085,9 @@ scipy==1.16.2 # via chem semantic-version==2.10.0 # via edx-drf-extensions -shapely==2.1.1 +shapely==2.1.2 # via -r requirements/edx/kernel.in -simplejson==3.20.1 +simplejson==3.20.2 # via # -r requirements/edx/kernel.in # sailthru-client @@ -1118,7 +1119,7 @@ slumber==0.7.1 # enterprise-integrated-channels sniffio==1.3.1 # via anyio -snowflake-connector-python==3.17.3 +snowflake-connector-python==3.18.0 # via edx-enterprise social-auth-app-django==5.4.1 # via @@ -1177,6 +1178,7 @@ typing-extensions==4.15.0 # beautifulsoup4 # django-countries # edx-opaque-keys + # grpcio # jwcrypto # pydantic # pydantic-core @@ -1185,7 +1187,7 @@ typing-extensions==4.15.0 # referencing # snowflake-connector-python # typing-inspection -typing-inspection==0.4.1 +typing-inspection==0.4.2 # via pydantic tzdata==2025.2 # via @@ -1216,7 +1218,7 @@ voluptuous==0.15.2 # via ora2 walrus==0.9.5 # via edx-event-bus-redis -wcwidth==0.2.13 +wcwidth==0.2.14 # via prompt-toolkit web-fragments==3.1.0 # via @@ -1273,7 +1275,7 @@ xmlsec==1.3.14 # python3-saml xss-utils==0.8.0 # via -r requirements/edx/kernel.in -yarl==1.20.1 +yarl==1.22.0 # via aiohttp zipp==3.23.0 # via importlib-metadata diff --git a/requirements/edx/coverage.txt b/requirements/edx/coverage.txt index 57a6416926..010306d68c 100644 --- a/requirements/edx/coverage.txt +++ b/requirements/edx/coverage.txt @@ -6,13 +6,13 @@ # chardet==5.2.0 # via diff-cover -coverage==7.10.6 +coverage==7.10.7 # via -r requirements/edx/coverage.in -diff-cover==9.6.0 +diff-cover==9.7.1 # via -r requirements/edx/coverage.in jinja2==3.1.6 # via diff-cover -markupsafe==3.0.2 +markupsafe==3.0.3 # via jinja2 pluggy==1.6.0 # via diff-cover diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 1007ceba5f..9df3845c2e 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -17,7 +17,7 @@ aiohappyeyeballs==2.6.1 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # aiohttp -aiohttp==3.12.15 +aiohttp==3.13.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -51,7 +51,7 @@ annotated-types==0.7.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # pydantic -anyio==4.10.0 +anyio==4.11.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -62,7 +62,7 @@ appdirs==1.4.4 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # fs -asgiref==3.9.1 +asgiref==3.10.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -81,7 +81,7 @@ astroid==3.3.11 # pylint # pylint-celery # sphinx-autoapi -attrs==25.3.0 +attrs==25.4.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -105,19 +105,19 @@ backoff==1.10.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # analytics-python -bcrypt==4.3.0 +bcrypt==5.0.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # paramiko -beautifulsoup4==4.13.5 +beautifulsoup4==4.14.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # openedx-forum # pydata-sphinx-theme # pynliner -billiard==4.2.1 +billiard==4.2.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -136,7 +136,7 @@ boto==2.49.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -boto3==1.40.31 +boto3==1.40.46 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -144,7 +144,7 @@ boto3==1.40.31 # fs-s3fs # ora2 # snowflake-connector-python -botocore==1.40.31 +botocore==1.40.46 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -164,7 +164,7 @@ cachecontrol==0.14.3 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # firebase-admin -cachetools==5.5.2 +cachetools==6.2.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -188,7 +188,7 @@ celery==5.5.3 # enterprise-integrated-channels # event-tracking # openedx-learning -certifi==2025.8.3 +certifi==2025.10.5 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -222,7 +222,7 @@ chem==2.0.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -click==8.2.1 +click==8.3.0 # via # -r requirements/edx/assets.txt # -r requirements/edx/development.in @@ -276,7 +276,7 @@ colorama==0.4.6 # via # -r requirements/edx/testing.txt # tox -coverage[toml]==7.10.6 +coverage[toml]==7.10.7 # via # -r requirements/edx/testing.txt # pytest-cov @@ -286,6 +286,7 @@ crowdsourcehinter-xblock==0.8 # -r requirements/edx/testing.txt cryptography==45.0.7 # via + # -c requirements/constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # django-fernet-fields-v2 @@ -320,7 +321,7 @@ defusedxml==0.7.1 # ora2 # python3-openid # social-auth-core -diff-cover==9.6.0 +diff-cover==9.7.1 # via -r requirements/edx/testing.txt dill==0.4.0 # via @@ -439,7 +440,7 @@ django-config-models==2.9.0 # edx-name-affirmation # enterprise-integrated-channels # lti-consumer-xblock -django-cors-headers==4.8.0 +django-cors-headers==4.9.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -519,7 +520,7 @@ django-multi-email-field==0.8.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-enterprise -django-mysql==4.18.0 +django-mysql==4.19.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -581,12 +582,12 @@ django-storages==1.14.6 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edxval -django-stubs[compatible-mypy]==5.2.5 +django-stubs[compatible-mypy]==5.2.6 # via # -c requirements/constraints.txt # -r requirements/edx/development.in # djangorestframework-stubs -django-stubs-ext==5.2.5 +django-stubs-ext==5.2.6 # via django-stubs django-user-tasks==3.4.3 # via @@ -627,7 +628,7 @@ djangorestframework==3.16.1 # openedx-learning # ora2 # super-csv -djangorestframework-stubs==3.16.2 +djangorestframework-stubs==3.16.4 # via -r requirements/edx/development.in djangorestframework-xml==2.0.0 # via @@ -658,7 +659,7 @@ drf-spectacular==0.28.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -drf-yasg==1.21.10 +drf-yasg==1.21.11 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -673,7 +674,7 @@ edx-api-doc-tools==2.1.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-name-affirmation -edx-auth-backends==4.6.0 +edx-auth-backends==4.6.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -712,7 +713,7 @@ edx-django-sites-extensions==5.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -edx-django-utils==8.0.0 +edx-django-utils==8.0.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -824,7 +825,7 @@ edx-sga==0.26.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -edx-submissions==3.11.1 +edx-submissions==3.12.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -875,7 +876,7 @@ enmerkar-underscore==2.4.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -enterprise-integrated-channels==0.1.16 +enterprise-integrated-channels==0.1.18 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -896,7 +897,7 @@ faker==37.8.0 # via # -r requirements/edx/testing.txt # factory-boy -fastapi==0.116.1 +fastapi==0.118.0 # via # -r requirements/edx/testing.txt # pact-python @@ -919,7 +920,7 @@ firebase-admin==7.1.0 # edx-ace freezegun==1.5.5 # via -r requirements/edx/testing.txt -frozenlist==1.7.0 +frozenlist==1.8.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -951,7 +952,7 @@ glob2==0.7 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -google-api-core[grpc]==2.25.1 +google-api-core[grpc]==2.25.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -959,7 +960,7 @@ google-api-core[grpc]==2.25.1 # google-cloud-core # google-cloud-firestore # google-cloud-storage -google-auth==2.40.3 +google-auth==2.41.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1004,13 +1005,13 @@ grimp==3.11 # via # -r requirements/edx/testing.txt # import-linter -grpcio==1.74.0 +grpcio==1.75.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # google-api-core # grpcio-status -grpcio-status==1.74.0 +grpcio-status==1.75.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1109,7 +1110,7 @@ isodate==0.7.2 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # python3-saml -isort==6.0.1 +isort==6.1.0 # via # -r requirements/edx/testing.txt # pylint @@ -1212,7 +1213,7 @@ lxml[html-clean]==5.3.2 # python3-saml # xblock # xmlsec -lxml-html-clean==0.4.2 +lxml-html-clean==0.4.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1236,7 +1237,7 @@ markdown==3.9 # openedx-django-wiki # staff-graded-xblock # xblock-poll -markupsafe==3.0.2 +markupsafe==3.0.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1289,13 +1290,13 @@ msgpack==1.1.1 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # cachecontrol -multidict==6.6.4 +multidict==6.7.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # aiohttp # yarl -mypy==1.18.1 +mypy==1.18.2 # via # -r requirements/edx/development.in # django-stubs @@ -1311,7 +1312,7 @@ nh3==0.3.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # xblocks-contrib -nltk==3.9.1 +nltk==3.9.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1423,7 +1424,9 @@ packaging==25.0 # sphinx # tox pact-python==2.3.3 - # via -r requirements/edx/testing.txt + # via + # -c requirements/constraints.txt + # -r requirements/edx/testing.txt paramiko==4.0.0 # via # -r requirements/edx/doc.txt @@ -1465,7 +1468,7 @@ pillow==11.3.0 # edx-enterprise # edx-organizations # edxval -pip-tools==7.5.0 +pip-tools==7.5.1 # via -r requirements/pip-tools.txt platformdirs==4.4.0 # via @@ -1492,7 +1495,7 @@ prompt-toolkit==3.0.52 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # click-repl -propcache==0.3.2 +propcache==0.4.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1513,7 +1516,7 @@ protobuf==6.32.1 # googleapis-common-protos # grpcio-status # proto-plus -psutil==7.0.0 +psutil==7.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1553,7 +1556,7 @@ pycryptodomex==3.23.0 # -r requirements/edx/testing.txt # edx-proctoring # lti-consumer-xblock -pydantic==2.11.9 +pydantic==2.11.10 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1596,7 +1599,7 @@ pylatexenc==2.10 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # olxcleaner -pylint==3.3.8 +pylint==3.3.9 # via # -r requirements/edx/testing.txt # edx-lint @@ -1646,12 +1649,12 @@ pynliner==0.8.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -pyopenssl==25.2.0 +pyopenssl==25.3.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # snowflake-connector-python -pyparsing==3.2.4 +pyparsing==3.2.5 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1766,7 +1769,7 @@ pyuca==1.2 # -r requirements/edx/testing.txt pywatchman==3.0.0 # via -r requirements/edx/development.in -pyyaml==6.0.2 +pyyaml==6.0.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1798,7 +1801,7 @@ referencing==0.36.2 # -r requirements/edx/testing.txt # jsonschema # jsonschema-specifications -regex==2025.9.1 +regex==2025.9.18 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1881,11 +1884,11 @@ semantic-version==2.10.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-drf-extensions -shapely==2.1.1 +shapely==2.1.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -simplejson==3.20.1 +simplejson==3.20.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1938,7 +1941,7 @@ snowballstemmer==3.0.1 # via # -r requirements/edx/doc.txt # sphinx -snowflake-connector-python==3.17.3 +snowflake-connector-python==3.18.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1982,7 +1985,7 @@ sphinx==8.2.3 # sphinxcontrib-httpdomain # sphinxcontrib-openapi # sphinxext-rediraffe -sphinx-autoapi==3.6.0 +sphinx-autoapi==3.6.1 # via -r requirements/edx/doc.txt sphinx-book-theme==1.1.4 # via -r requirements/edx/doc.txt @@ -2024,7 +2027,7 @@ sphinxcontrib-serializinghtml==2.0.0 # via # -r requirements/edx/doc.txt # sphinx -sphinxext-rediraffe==0.2.7 +sphinxext-rediraffe==0.3.0 # via -r requirements/edx/doc.txt sqlparse==0.5.3 # via @@ -2036,7 +2039,7 @@ staff-graded-xblock==3.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -starlette==0.47.3 +starlette==0.48.0 # via # -r requirements/edx/testing.txt # fastapi @@ -2081,7 +2084,7 @@ tomlkit==0.13.3 # openedx-learning # pylint # snowflake-connector-python -tox==4.27.0 +tox==4.30.3 # via -r requirements/edx/testing.txt tqdm==4.67.1 # via @@ -2109,6 +2112,7 @@ typing-extensions==4.15.0 # edx-opaque-keys # fastapi # grimp + # grpcio # import-linter # jwcrypto # mypy @@ -2121,7 +2125,7 @@ typing-extensions==4.15.0 # snowflake-connector-python # starlette # typing-inspection -typing-inspection==0.4.1 +typing-inspection==0.4.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2159,7 +2163,7 @@ urllib3==2.5.0 # elasticsearch # requests # types-requests -uvicorn==0.35.0 +uvicorn==0.37.0 # via # -r requirements/edx/testing.txt # pact-python @@ -2188,7 +2192,7 @@ walrus==0.9.5 # edx-event-bus-redis watchdog==6.0.0 # via -r requirements/edx/development.in -wcwidth==0.2.13 +wcwidth==0.2.14 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2274,7 +2278,7 @@ xss-utils==0.8.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -yarl==1.20.1 +yarl==1.22.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 7407af9f36..faba8969f1 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -12,7 +12,7 @@ aiohappyeyeballs==2.6.1 # via # -r requirements/edx/base.txt # aiohttp -aiohttp==3.12.15 +aiohttp==3.13.0 # via # -r requirements/edx/base.txt # geoip2 @@ -37,7 +37,7 @@ annotated-types==0.7.0 # via # -r requirements/edx/base.txt # pydantic -anyio==4.10.0 +anyio==4.11.0 # via # -r requirements/edx/base.txt # httpx @@ -45,7 +45,7 @@ appdirs==1.4.4 # via # -r requirements/edx/base.txt # fs -asgiref==3.9.1 +asgiref==3.10.0 # via # -r requirements/edx/base.txt # django @@ -57,7 +57,7 @@ asn1crypto==1.5.1 # snowflake-connector-python astroid==3.3.11 # via sphinx-autoapi -attrs==25.3.0 +attrs==25.4.0 # via # -r requirements/edx/base.txt # aiohttp @@ -78,17 +78,17 @@ backoff==1.10.0 # via # -r requirements/edx/base.txt # analytics-python -bcrypt==4.3.0 +bcrypt==5.0.0 # via # -r requirements/edx/base.txt # paramiko -beautifulsoup4==4.13.5 +beautifulsoup4==4.14.2 # via # -r requirements/edx/base.txt # openedx-forum # pydata-sphinx-theme # pynliner -billiard==4.2.1 +billiard==4.2.2 # via # -r requirements/edx/base.txt # celery @@ -103,14 +103,14 @@ bleach[css]==6.2.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/base.txt -boto3==1.40.31 +boto3==1.40.46 # via # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 # snowflake-connector-python -botocore==1.40.31 +botocore==1.40.46 # via # -r requirements/edx/base.txt # boto3 @@ -122,7 +122,7 @@ cachecontrol==0.14.3 # via # -r requirements/edx/base.txt # firebase-admin -cachetools==5.5.2 +cachetools==6.2.0 # via # -r requirements/edx/base.txt # edxval @@ -142,7 +142,7 @@ celery==5.5.3 # enterprise-integrated-channels # event-tracking # openedx-learning -certifi==2025.8.3 +certifi==2025.10.5 # via # -r requirements/edx/base.txt # elasticsearch @@ -167,7 +167,7 @@ charset-normalizer==3.4.3 # snowflake-connector-python chem==2.0.0 # via -r requirements/edx/base.txt -click==8.2.1 +click==8.3.0 # via # -r requirements/edx/base.txt # celery @@ -201,6 +201,7 @@ crowdsourcehinter-xblock==0.8 # via -r requirements/edx/base.txt cryptography==45.0.7 # via + # -c requirements/constraints.txt # -r requirements/edx/base.txt # django-fernet-fields-v2 # edx-enterprise @@ -321,7 +322,7 @@ django-config-models==2.9.0 # edx-name-affirmation # enterprise-integrated-channels # lti-consumer-xblock -django-cors-headers==4.8.0 +django-cors-headers==4.9.0 # via -r requirements/edx/base.txt django-countries==7.6.1 # via @@ -384,7 +385,7 @@ django-multi-email-field==0.8.0 # via # -r requirements/edx/base.txt # edx-enterprise -django-mysql==4.18.0 +django-mysql==4.19.0 # via -r requirements/edx/base.txt django-oauth-toolkit==1.7.1 # via @@ -486,7 +487,7 @@ drf-jwt==1.19.2 # edx-drf-extensions drf-spectacular==0.28.0 # via -r requirements/edx/base.txt -drf-yasg==1.21.10 +drf-yasg==1.21.11 # via # -r requirements/edx/base.txt # django-user-tasks @@ -497,7 +498,7 @@ edx-api-doc-tools==2.1.0 # via # -r requirements/edx/base.txt # edx-name-affirmation -edx-auth-backends==4.6.0 +edx-auth-backends==4.6.1 # via -r requirements/edx/base.txt edx-bulk-grades==1.2.0 # via @@ -524,7 +525,7 @@ edx-django-release-util==1.5.0 # edxval edx-django-sites-extensions==5.1.0 # via -r requirements/edx/base.txt -edx-django-utils==8.0.0 +edx-django-utils==8.0.1 # via # -r requirements/edx/base.txt # django-config-models @@ -612,7 +613,7 @@ edx-search==4.3.0 # openedx-forum edx-sga==0.26.0 # via -r requirements/edx/base.txt -edx-submissions==3.11.1 +edx-submissions==3.12.0 # via # -r requirements/edx/base.txt # ora2 @@ -653,7 +654,7 @@ enmerkar==0.7.1 # enmerkar-underscore enmerkar-underscore==2.4.0 # via -r requirements/edx/base.txt -enterprise-integrated-channels==0.1.16 +enterprise-integrated-channels==0.1.18 # via -r requirements/edx/base.txt event-tracking==3.3.0 # via @@ -673,7 +674,7 @@ firebase-admin==7.1.0 # via # -r requirements/edx/base.txt # edx-ace -frozenlist==1.7.0 +frozenlist==1.8.0 # via # -r requirements/edx/base.txt # aiohttp @@ -696,14 +697,14 @@ gitpython==3.1.45 # via -r requirements/edx/doc.in glob2==0.7 # via -r requirements/edx/base.txt -google-api-core[grpc]==2.25.1 +google-api-core[grpc]==2.25.2 # via # -r requirements/edx/base.txt # firebase-admin # google-cloud-core # google-cloud-firestore # google-cloud-storage -google-auth==2.40.3 +google-auth==2.41.1 # via # -r requirements/edx/base.txt # google-api-core @@ -737,12 +738,12 @@ googleapis-common-protos==1.70.0 # -r requirements/edx/base.txt # google-api-core # grpcio-status -grpcio==1.74.0 +grpcio==1.75.1 # via # -r requirements/edx/base.txt # google-api-core # grpcio-status -grpcio-status==1.74.0 +grpcio-status==1.75.1 # via # -r requirements/edx/base.txt # google-api-core @@ -885,7 +886,7 @@ lxml[html-clean]==5.3.2 # python3-saml # xblock # xmlsec -lxml-html-clean==0.4.2 +lxml-html-clean==0.4.3 # via # -r requirements/edx/base.txt # lxml @@ -904,7 +905,7 @@ markdown==3.9 # openedx-django-wiki # staff-graded-xblock # xblock-poll -markupsafe==3.0.2 +markupsafe==3.0.3 # via # -r requirements/edx/base.txt # chem @@ -940,7 +941,7 @@ msgpack==1.1.1 # via # -r requirements/edx/base.txt # cachecontrol -multidict==6.6.4 +multidict==6.7.0 # via # -r requirements/edx/base.txt # aiohttp @@ -953,7 +954,7 @@ nh3==0.3.0 # via # -r requirements/edx/base.txt # xblocks-contrib -nltk==3.9.1 +nltk==3.9.2 # via # -r requirements/edx/base.txt # chem @@ -1074,7 +1075,7 @@ prompt-toolkit==3.0.52 # via # -r requirements/edx/base.txt # click-repl -propcache==0.3.2 +propcache==0.4.0 # via # -r requirements/edx/base.txt # aiohttp @@ -1092,7 +1093,7 @@ protobuf==6.32.1 # googleapis-common-protos # grpcio-status # proto-plus -psutil==7.0.0 +psutil==7.1.0 # via # -r requirements/edx/base.txt # edx-django-utils @@ -1117,7 +1118,7 @@ pycryptodomex==3.23.0 # -r requirements/edx/base.txt # edx-proctoring # lti-consumer-xblock -pydantic==2.11.9 +pydantic==2.11.10 # via # -r requirements/edx/base.txt # camel-converter @@ -1169,11 +1170,11 @@ pynacl==1.6.0 # paramiko pynliner==0.8.0 # via -r requirements/edx/base.txt -pyopenssl==25.2.0 +pyopenssl==25.3.0 # via # -r requirements/edx/base.txt # snowflake-connector-python -pyparsing==3.2.4 +pyparsing==3.2.5 # via # -r requirements/edx/base.txt # chem @@ -1234,7 +1235,7 @@ pytz==2025.2 # xblock pyuca==1.2 # via -r requirements/edx/base.txt -pyyaml==6.0.2 +pyyaml==6.0.3 # via # -r requirements/edx/base.txt # code-annotations @@ -1259,7 +1260,7 @@ referencing==0.36.2 # -r requirements/edx/base.txt # jsonschema # jsonschema-specifications -regex==2025.9.1 +regex==2025.9.18 # via # -r requirements/edx/base.txt # nltk @@ -1328,9 +1329,9 @@ semantic-version==2.10.0 # via # -r requirements/edx/base.txt # edx-drf-extensions -shapely==2.1.1 +shapely==2.1.2 # via -r requirements/edx/base.txt -simplejson==3.20.1 +simplejson==3.20.2 # via # -r requirements/edx/base.txt # sailthru-client @@ -1369,7 +1370,7 @@ sniffio==1.3.1 # anyio snowballstemmer==3.0.1 # via sphinx -snowflake-connector-python==3.17.3 +snowflake-connector-python==3.18.0 # via # -r requirements/edx/base.txt # edx-enterprise @@ -1407,7 +1408,7 @@ sphinx==8.2.3 # sphinxcontrib-httpdomain # sphinxcontrib-openapi # sphinxext-rediraffe -sphinx-autoapi==3.6.0 +sphinx-autoapi==3.6.1 # via -r requirements/edx/doc.in sphinx-book-theme==1.1.4 # via -r requirements/edx/doc.in @@ -1433,7 +1434,7 @@ sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 # via sphinx -sphinxext-rediraffe==0.2.7 +sphinxext-rediraffe==0.3.0 # via -r requirements/edx/doc.in sqlparse==0.5.3 # via @@ -1487,6 +1488,7 @@ typing-extensions==4.15.0 # beautifulsoup4 # django-countries # edx-opaque-keys + # grpcio # jwcrypto # pydantic # pydantic-core @@ -1496,7 +1498,7 @@ typing-extensions==4.15.0 # referencing # snowflake-connector-python # typing-inspection -typing-inspection==0.4.1 +typing-inspection==0.4.2 # via # -r requirements/edx/base.txt # pydantic @@ -1537,7 +1539,7 @@ walrus==0.9.5 # via # -r requirements/edx/base.txt # edx-event-bus-redis -wcwidth==0.2.13 +wcwidth==0.2.14 # via # -r requirements/edx/base.txt # prompt-toolkit @@ -1601,7 +1603,7 @@ xmlsec==1.3.14 # python3-saml xss-utils==0.8.0 # via -r requirements/edx/base.txt -yarl==1.20.1 +yarl==1.22.0 # via # -r requirements/edx/base.txt # aiohttp diff --git a/requirements/edx/semgrep.txt b/requirements/edx/semgrep.txt index aec4c59acb..6adeb975ef 100644 --- a/requirements/edx/semgrep.txt +++ b/requirements/edx/semgrep.txt @@ -4,7 +4,15 @@ # # make upgrade # -attrs==25.3.0 +annotated-types==0.7.0 + # via pydantic +anyio==4.11.0 + # via + # httpx + # mcp + # sse-starlette + # starlette +attrs==25.4.0 # via # glom # jsonschema @@ -17,24 +25,24 @@ boltons==21.0.0 # semgrep bracex==2.6 # via wcmatch -certifi==2025.8.3 - # via requests +certifi==2025.10.5 + # via + # httpcore + # httpx + # requests charset-normalizer==3.4.3 # via requests click==8.1.8 # via # click-option-group # semgrep -click-option-group==0.5.7 + # uvicorn +click-option-group==0.5.8 # via semgrep colorama==0.4.6 # via semgrep defusedxml==0.7.1 # via semgrep -deprecated==1.2.18 - # via - # opentelemetry-api - # opentelemetry-exporter-otlp-proto-http exceptiongroup==1.2.2 # via semgrep face==24.0.0 @@ -43,19 +51,36 @@ glom==22.1.0 # via semgrep googleapis-common-protos==1.70.0 # via opentelemetry-exporter-otlp-proto-http +h11==0.16.0 + # via + # httpcore + # uvicorn +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via mcp +httpx-sse==0.4.1 + # via mcp idna==3.10 - # via requests -importlib-metadata==7.1.0 + # via + # anyio + # httpx + # requests +importlib-metadata==8.7.0 # via opentelemetry-api -jsonschema==4.25.1 - # via semgrep +jsonschema==4.20.0 + # via + # mcp + # semgrep jsonschema-specifications==2025.9.1 # via jsonschema markdown-it-py==4.0.0 # via rich +mcp==1.12.2 + # via semgrep mdurl==0.1.2 # via markdown-it-py -opentelemetry-api==1.25.0 +opentelemetry-api==1.37.0 # via # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation @@ -63,38 +88,53 @@ opentelemetry-api==1.25.0 # opentelemetry-sdk # opentelemetry-semantic-conventions # semgrep -opentelemetry-exporter-otlp-proto-common==1.25.0 +opentelemetry-exporter-otlp-proto-common==1.37.0 # via opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-http==1.25.0 +opentelemetry-exporter-otlp-proto-http==1.37.0 # via semgrep -opentelemetry-instrumentation==0.46b0 +opentelemetry-instrumentation==0.58b0 # via opentelemetry-instrumentation-requests -opentelemetry-instrumentation-requests==0.46b0 +opentelemetry-instrumentation-requests==0.58b0 # via semgrep -opentelemetry-proto==1.25.0 +opentelemetry-proto==1.37.0 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.25.0 +opentelemetry-sdk==1.37.0 # via # opentelemetry-exporter-otlp-proto-http # semgrep -opentelemetry-semantic-conventions==0.46b0 +opentelemetry-semantic-conventions==0.58b0 # via + # opentelemetry-instrumentation # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.46b0 +opentelemetry-util-http==0.58b0 # via opentelemetry-instrumentation-requests packaging==25.0 - # via semgrep + # via + # opentelemetry-instrumentation + # semgrep peewee==3.18.2 # via semgrep -protobuf==4.25.8 +protobuf==6.32.1 # via # googleapis-common-protos # opentelemetry-proto +pydantic==2.11.10 + # via + # mcp + # pydantic-settings +pydantic-core==2.33.2 + # via pydantic +pydantic-settings==2.11.0 + # via mcp pygments==2.19.2 # via rich +python-dotenv==1.1.1 + # via pydantic-settings +python-multipart==0.0.20 + # via mcp referencing==0.36.2 # via # jsonschema @@ -112,28 +152,45 @@ rpds-py==0.27.1 ruamel-yaml==0.18.15 # via semgrep ruamel-yaml-clib==0.2.12 - # via ruamel-yaml -semgrep==1.136.0 + # via + # ruamel-yaml + # semgrep +semgrep==1.139.0 # via -r requirements/edx/semgrep.in +sniffio==1.3.1 + # via anyio +sse-starlette==3.0.2 + # via mcp +starlette==0.48.0 + # via mcp tomli==2.0.2 # via semgrep typing-extensions==4.15.0 # via + # anyio + # opentelemetry-api + # opentelemetry-exporter-otlp-proto-http # opentelemetry-sdk + # opentelemetry-semantic-conventions + # pydantic + # pydantic-core # referencing # semgrep + # starlette + # typing-inspection +typing-inspection==0.4.2 + # via + # pydantic + # pydantic-settings urllib3==2.5.0 # via # requests # semgrep +uvicorn==0.37.0 + # via mcp wcmatch==8.5.2 # via semgrep wrapt==1.17.3 - # via - # deprecated - # opentelemetry-instrumentation + # via opentelemetry-instrumentation zipp==3.23.0 # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 319d22004b..7de977c716 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -10,7 +10,7 @@ aiohappyeyeballs==2.6.1 # via # -r requirements/edx/base.txt # aiohttp -aiohttp==3.12.15 +aiohttp==3.13.0 # via # -r requirements/edx/base.txt # geoip2 @@ -33,7 +33,7 @@ annotated-types==0.7.0 # via # -r requirements/edx/base.txt # pydantic -anyio==4.10.0 +anyio==4.11.0 # via # -r requirements/edx/base.txt # httpx @@ -42,7 +42,7 @@ appdirs==1.4.4 # via # -r requirements/edx/base.txt # fs -asgiref==3.9.1 +asgiref==3.10.0 # via # -r requirements/edx/base.txt # django @@ -56,7 +56,7 @@ astroid==3.3.11 # via # pylint # pylint-celery -attrs==25.3.0 +attrs==25.4.0 # via # -r requirements/edx/base.txt # aiohttp @@ -75,17 +75,17 @@ backoff==1.10.0 # via # -r requirements/edx/base.txt # analytics-python -bcrypt==4.3.0 +bcrypt==5.0.0 # via # -r requirements/edx/base.txt # paramiko -beautifulsoup4==4.13.5 +beautifulsoup4==4.14.2 # via # -r requirements/edx/base.txt # -r requirements/edx/testing.in # openedx-forum # pynliner -billiard==4.2.1 +billiard==4.2.2 # via # -r requirements/edx/base.txt # celery @@ -100,14 +100,14 @@ bleach[css]==6.2.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/base.txt -boto3==1.40.31 +boto3==1.40.46 # via # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 # snowflake-connector-python -botocore==1.40.31 +botocore==1.40.46 # via # -r requirements/edx/base.txt # boto3 @@ -119,7 +119,7 @@ cachecontrol==0.14.3 # via # -r requirements/edx/base.txt # firebase-admin -cachetools==5.5.2 +cachetools==6.2.0 # via # -r requirements/edx/base.txt # edxval @@ -140,7 +140,7 @@ celery==5.5.3 # enterprise-integrated-channels # event-tracking # openedx-learning -certifi==2025.8.3 +certifi==2025.10.5 # via # -r requirements/edx/base.txt # elasticsearch @@ -169,7 +169,7 @@ charset-normalizer==3.4.3 # snowflake-connector-python chem==2.0.0 # via -r requirements/edx/base.txt -click==8.2.1 +click==8.3.0 # via # -r requirements/edx/base.txt # celery @@ -209,7 +209,7 @@ codejail-includes==2.0.0 # via -r requirements/edx/base.txt colorama==0.4.6 # via tox -coverage[toml]==7.10.6 +coverage[toml]==7.10.7 # via # -r requirements/edx/coverage.txt # pytest-cov @@ -217,6 +217,7 @@ crowdsourcehinter-xblock==0.8 # via -r requirements/edx/base.txt cryptography==45.0.7 # via + # -c requirements/constraints.txt # -r requirements/edx/base.txt # django-fernet-fields-v2 # edx-enterprise @@ -244,7 +245,7 @@ defusedxml==0.7.1 # ora2 # python3-openid # social-auth-core -diff-cover==9.6.0 +diff-cover==9.7.1 # via -r requirements/edx/coverage.txt dill==0.4.0 # via pylint @@ -347,7 +348,7 @@ django-config-models==2.9.0 # edx-name-affirmation # enterprise-integrated-channels # lti-consumer-xblock -django-cors-headers==4.8.0 +django-cors-headers==4.9.0 # via -r requirements/edx/base.txt django-countries==7.6.1 # via @@ -410,7 +411,7 @@ django-multi-email-field==0.8.0 # via # -r requirements/edx/base.txt # edx-enterprise -django-mysql==4.18.0 +django-mysql==4.19.0 # via -r requirements/edx/base.txt django-oauth-toolkit==1.7.1 # via @@ -507,7 +508,7 @@ drf-jwt==1.19.2 # edx-drf-extensions drf-spectacular==0.28.0 # via -r requirements/edx/base.txt -drf-yasg==1.21.10 +drf-yasg==1.21.11 # via # -r requirements/edx/base.txt # django-user-tasks @@ -518,7 +519,7 @@ edx-api-doc-tools==2.1.0 # via # -r requirements/edx/base.txt # edx-name-affirmation -edx-auth-backends==4.6.0 +edx-auth-backends==4.6.1 # via -r requirements/edx/base.txt edx-bulk-grades==1.2.0 # via @@ -545,7 +546,7 @@ edx-django-release-util==1.5.0 # edxval edx-django-sites-extensions==5.1.0 # via -r requirements/edx/base.txt -edx-django-utils==8.0.0 +edx-django-utils==8.0.1 # via # -r requirements/edx/base.txt # django-config-models @@ -635,7 +636,7 @@ edx-search==4.3.0 # openedx-forum edx-sga==0.26.0 # via -r requirements/edx/base.txt -edx-submissions==3.11.1 +edx-submissions==3.12.0 # via # -r requirements/edx/base.txt # ora2 @@ -676,7 +677,7 @@ enmerkar==0.7.1 # enmerkar-underscore enmerkar-underscore==2.4.0 # via -r requirements/edx/base.txt -enterprise-integrated-channels==0.1.16 +enterprise-integrated-channels==0.1.18 # via -r requirements/edx/base.txt event-tracking==3.3.0 # via @@ -690,7 +691,7 @@ factory-boy==3.3.3 # via -r requirements/edx/testing.in faker==37.8.0 # via factory-boy -fastapi==0.116.1 +fastapi==0.118.0 # via pact-python fastavro==1.12.0 # via @@ -708,7 +709,7 @@ firebase-admin==7.1.0 # edx-ace freezegun==1.5.5 # via -r requirements/edx/testing.in -frozenlist==1.7.0 +frozenlist==1.8.0 # via # -r requirements/edx/base.txt # aiohttp @@ -727,14 +728,14 @@ geoip2==5.1.0 # via -r requirements/edx/base.txt glob2==0.7 # via -r requirements/edx/base.txt -google-api-core[grpc]==2.25.1 +google-api-core[grpc]==2.25.2 # via # -r requirements/edx/base.txt # firebase-admin # google-cloud-core # google-cloud-firestore # google-cloud-storage -google-auth==2.40.3 +google-auth==2.41.1 # via # -r requirements/edx/base.txt # google-api-core @@ -770,12 +771,12 @@ googleapis-common-protos==1.70.0 # grpcio-status grimp==3.11 # via import-linter -grpcio==1.74.0 +grpcio==1.75.1 # via # -r requirements/edx/base.txt # google-api-core # grpcio-status -grpcio-status==1.74.0 +grpcio-status==1.75.1 # via # -r requirements/edx/base.txt # google-api-core @@ -846,7 +847,7 @@ isodate==0.7.2 # via # -r requirements/edx/base.txt # python3-saml -isort==6.0.1 +isort==6.1.0 # via # -r requirements/edx/testing.in # pylint @@ -927,7 +928,7 @@ lxml[html-clean]==5.3.2 # python3-saml # xblock # xmlsec -lxml-html-clean==0.4.2 +lxml-html-clean==0.4.3 # via # -r requirements/edx/base.txt # lxml @@ -946,7 +947,7 @@ markdown==3.9 # openedx-django-wiki # staff-graded-xblock # xblock-poll -markupsafe==3.0.2 +markupsafe==3.0.3 # via # -r requirements/edx/base.txt # -r requirements/edx/coverage.txt @@ -985,7 +986,7 @@ msgpack==1.1.1 # via # -r requirements/edx/base.txt # cachecontrol -multidict==6.6.4 +multidict==6.7.0 # via # -r requirements/edx/base.txt # aiohttp @@ -998,7 +999,7 @@ nh3==0.3.0 # via # -r requirements/edx/base.txt # xblocks-contrib -nltk==3.9.1 +nltk==3.9.2 # via # -r requirements/edx/base.txt # chem @@ -1079,7 +1080,9 @@ packaging==25.0 # snowflake-connector-python # tox pact-python==2.3.3 - # via -r requirements/edx/testing.in + # via + # -c requirements/constraints.txt + # -r requirements/edx/testing.in paramiko==4.0.0 # via # -r requirements/edx/base.txt @@ -1131,7 +1134,7 @@ prompt-toolkit==3.0.52 # via # -r requirements/edx/base.txt # click-repl -propcache==0.3.2 +propcache==0.4.0 # via # -r requirements/edx/base.txt # aiohttp @@ -1149,7 +1152,7 @@ protobuf==6.32.1 # googleapis-common-protos # grpcio-status # proto-plus -psutil==7.0.0 +psutil==7.1.0 # via # -r requirements/edx/base.txt # edx-django-utils @@ -1182,7 +1185,7 @@ pycryptodomex==3.23.0 # -r requirements/edx/base.txt # edx-proctoring # lti-consumer-xblock -pydantic==2.11.9 +pydantic==2.11.10 # via # -r requirements/edx/base.txt # camel-converter @@ -1212,7 +1215,7 @@ pylatexenc==2.10 # via # -r requirements/edx/base.txt # olxcleaner -pylint==3.3.8 +pylint==3.3.9 # via # edx-lint # pylint-celery @@ -1248,11 +1251,11 @@ pynacl==1.6.0 # paramiko pynliner==0.8.0 # via -r requirements/edx/base.txt -pyopenssl==25.2.0 +pyopenssl==25.3.0 # via # -r requirements/edx/base.txt # snowflake-connector-python -pyparsing==3.2.4 +pyparsing==3.2.5 # via # -r requirements/edx/base.txt # chem @@ -1345,7 +1348,7 @@ pytz==2025.2 # xblock pyuca==1.2 # via -r requirements/edx/base.txt -pyyaml==6.0.2 +pyyaml==6.0.3 # via # -r requirements/edx/base.txt # code-annotations @@ -1368,7 +1371,7 @@ referencing==0.36.2 # -r requirements/edx/base.txt # jsonschema # jsonschema-specifications -regex==2025.9.1 +regex==2025.9.18 # via # -r requirements/edx/base.txt # nltk @@ -1435,9 +1438,9 @@ semantic-version==2.10.0 # via # -r requirements/edx/base.txt # edx-drf-extensions -shapely==2.1.1 +shapely==2.1.2 # via -r requirements/edx/base.txt -simplejson==3.20.1 +simplejson==3.20.2 # via # -r requirements/edx/base.txt # sailthru-client @@ -1475,7 +1478,7 @@ sniffio==1.3.1 # via # -r requirements/edx/base.txt # anyio -snowflake-connector-python==3.17.3 +snowflake-connector-python==3.18.0 # via # -r requirements/edx/base.txt # edx-enterprise @@ -1507,7 +1510,7 @@ sqlparse==0.5.3 # django staff-graded-xblock==3.1.0 # via -r requirements/edx/base.txt -starlette==0.47.3 +starlette==0.48.0 # via fastapi stevedore==5.5.0 # via @@ -1544,7 +1547,7 @@ tomlkit==0.13.3 # openedx-learning # pylint # snowflake-connector-python -tox==4.27.0 +tox==4.30.3 # via -r requirements/edx/testing.in tqdm==4.67.1 # via @@ -1561,6 +1564,7 @@ typing-extensions==4.15.0 # edx-opaque-keys # fastapi # grimp + # grpcio # import-linter # jwcrypto # pydantic @@ -1571,7 +1575,7 @@ typing-extensions==4.15.0 # snowflake-connector-python # starlette # typing-inspection -typing-inspection==0.4.1 +typing-inspection==0.4.2 # via # -r requirements/edx/base.txt # pydantic @@ -1601,7 +1605,7 @@ urllib3==2.5.0 # botocore # elasticsearch # requests -uvicorn==0.35.0 +uvicorn==0.37.0 # via pact-python vine==5.1.0 # via @@ -1619,7 +1623,7 @@ walrus==0.9.5 # via # -r requirements/edx/base.txt # edx-event-bus-redis -wcwidth==0.2.13 +wcwidth==0.2.14 # via # -r requirements/edx/base.txt # prompt-toolkit @@ -1683,7 +1687,7 @@ xmlsec==1.3.14 # python3-saml xss-utils==0.8.0 # via -r requirements/edx/base.txt -yarl==1.20.1 +yarl==1.22.0 # via # -r requirements/edx/base.txt # aiohttp diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index b19a4faaa0..e97cb1b3d3 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -6,11 +6,11 @@ # build==1.3.0 # via pip-tools -click==8.2.1 +click==8.3.0 # via pip-tools packaging==25.0 # via build -pip-tools==7.5.0 +pip-tools==7.5.1 # via -r requirements/pip-tools.in pyproject-hooks==1.2.0 # via diff --git a/scripts/structures_pruning/requirements/base.txt b/scripts/structures_pruning/requirements/base.txt index a01b730c49..aa4a8c1576 100644 --- a/scripts/structures_pruning/requirements/base.txt +++ b/scripts/structures_pruning/requirements/base.txt @@ -4,7 +4,7 @@ # # make upgrade # -click==8.2.1 +click==8.3.0 # via # -r scripts/structures_pruning/requirements/base.in # click-log diff --git a/scripts/structures_pruning/requirements/testing.txt b/scripts/structures_pruning/requirements/testing.txt index 83d3f5746e..1b387d33c4 100644 --- a/scripts/structures_pruning/requirements/testing.txt +++ b/scripts/structures_pruning/requirements/testing.txt @@ -4,7 +4,7 @@ # # make upgrade # -click==8.2.1 +click==8.3.0 # via # -r scripts/structures_pruning/requirements/base.txt # click-log diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index b77e368467..fd67805f02 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -4,21 +4,21 @@ # # make upgrade # -asgiref==3.9.1 +asgiref==3.10.0 # via django -attrs==25.3.0 +attrs==25.4.0 # via zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.in -boto3==1.40.31 +boto3==1.40.46 # via -r scripts/user_retirement/requirements/base.in -botocore==1.40.31 +botocore==1.40.46 # via # boto3 # s3transfer -cachetools==5.5.2 +cachetools==6.2.0 # via google-auth -certifi==2025.8.3 +certifi==2025.10.5 # via requests cffi==2.0.0 # via @@ -26,12 +26,14 @@ cffi==2.0.0 # pynacl charset-normalizer==3.4.3 # via requests -click==8.2.1 +click==8.3.0 # via # -r scripts/user_retirement/requirements/base.in # edx-django-utils cryptography==45.0.7 - # via pyjwt + # via + # -c requirements/constraints.txt + # pyjwt django==4.2.25 # via # -c requirements/constraints.txt @@ -42,15 +44,15 @@ django-crum==0.7.9 # via edx-django-utils django-waffle==5.0.0 # via edx-django-utils -edx-django-utils==8.0.0 +edx-django-utils==8.0.1 # via edx-rest-api-client edx-rest-api-client==6.2.0 # via -r scripts/user_retirement/requirements/base.in -google-api-core==2.25.1 +google-api-core==2.25.2 # via google-api-python-client -google-api-python-client==2.181.0 +google-api-python-client==2.184.0 # via -r scripts/user_retirement/requirements/base.in -google-auth==2.40.3 +google-auth==2.41.1 # via # google-api-core # google-api-python-client @@ -88,7 +90,7 @@ protobuf==6.32.1 # google-api-core # googleapis-common-protos # proto-plus -psutil==7.0.0 +psutil==7.1.0 # via edx-django-utils pyasn1==0.6.1 # via @@ -104,7 +106,7 @@ pyjwt[crypto]==2.10.1 # simple-salesforce pynacl==1.6.0 # via edx-django-utils -pyparsing==3.2.4 +pyparsing==3.2.5 # via httplib2 python-dateutil==2.9.0.post0 # via botocore @@ -112,7 +114,7 @@ pytz==2025.2 # via # jenkinsapi # zeep -pyyaml==6.0.2 +pyyaml==6.0.3 # via -r scripts/user_retirement/requirements/base.in requests==2.32.5 # via @@ -134,7 +136,7 @@ s3transfer==0.14.0 # via boto3 simple-salesforce==1.12.9 # via -r scripts/user_retirement/requirements/base.in -simplejson==3.20.1 +simplejson==3.20.2 # via -r scripts/user_retirement/requirements/base.in six==1.17.0 # via python-dateutil diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index f0373c1817..31b20fc6d8 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -4,31 +4,31 @@ # # make upgrade # -asgiref==3.9.1 +asgiref==3.10.0 # via # -r scripts/user_retirement/requirements/base.txt # django -attrs==25.3.0 +attrs==25.4.0 # via # -r scripts/user_retirement/requirements/base.txt # zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.txt -boto3==1.40.31 +boto3==1.40.46 # via # -r scripts/user_retirement/requirements/base.txt # moto -botocore==1.40.31 +botocore==1.40.46 # via # -r scripts/user_retirement/requirements/base.txt # boto3 # moto # s3transfer -cachetools==5.5.2 +cachetools==6.2.0 # via # -r scripts/user_retirement/requirements/base.txt # google-auth -certifi==2025.8.3 +certifi==2025.10.5 # via # -r scripts/user_retirement/requirements/base.txt # requests @@ -41,7 +41,7 @@ charset-normalizer==3.4.3 # via # -r scripts/user_retirement/requirements/base.txt # requests -click==8.2.1 +click==8.3.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils @@ -66,19 +66,19 @@ django-waffle==5.0.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils -edx-django-utils==8.0.0 +edx-django-utils==8.0.1 # via # -r scripts/user_retirement/requirements/base.txt # edx-rest-api-client edx-rest-api-client==6.2.0 # via -r scripts/user_retirement/requirements/base.txt -google-api-core==2.25.1 +google-api-core==2.25.2 # via # -r scripts/user_retirement/requirements/base.txt # google-api-python-client -google-api-python-client==2.181.0 +google-api-python-client==2.184.0 # via -r scripts/user_retirement/requirements/base.txt -google-auth==2.40.3 +google-auth==2.41.1 # via # -r scripts/user_retirement/requirements/base.txt # google-api-core @@ -120,7 +120,7 @@ lxml==5.3.2 # via # -r scripts/user_retirement/requirements/base.txt # zeep -markupsafe==3.0.2 +markupsafe==3.0.3 # via # jinja2 # werkzeug @@ -130,7 +130,7 @@ more-itertools==10.8.0 # via # -r scripts/user_retirement/requirements/base.txt # simple-salesforce -moto==5.1.12 +moto==5.1.14 # via -r scripts/user_retirement/requirements/testing.in packaging==25.0 # via pytest @@ -150,7 +150,7 @@ protobuf==6.32.1 # google-api-core # googleapis-common-protos # proto-plus -psutil==7.0.0 +psutil==7.1.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils @@ -178,7 +178,7 @@ pynacl==1.6.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils -pyparsing==3.2.4 +pyparsing==3.2.5 # via # -r scripts/user_retirement/requirements/base.txt # httplib2 @@ -194,7 +194,7 @@ pytz==2025.2 # -r scripts/user_retirement/requirements/base.txt # jenkinsapi # zeep -pyyaml==6.0.2 +pyyaml==6.0.3 # via # -r scripts/user_retirement/requirements/base.txt # responses @@ -235,7 +235,7 @@ s3transfer==0.14.0 # boto3 simple-salesforce==1.12.9 # via -r scripts/user_retirement/requirements/base.txt -simplejson==3.20.1 +simplejson==3.20.2 # via -r scripts/user_retirement/requirements/base.txt six==1.17.0 # via @@ -268,7 +268,7 @@ urllib3==2.5.0 # responses werkzeug==3.1.3 # via moto -xmltodict==1.0.0 +xmltodict==1.0.2 # via moto zeep==4.3.2 # via diff --git a/scripts/xblock/requirements.txt b/scripts/xblock/requirements.txt index 52fb237cb8..23d2ac5b8e 100644 --- a/scripts/xblock/requirements.txt +++ b/scripts/xblock/requirements.txt @@ -4,7 +4,7 @@ # # make upgrade # -certifi==2025.8.3 +certifi==2025.10.5 # via requests charset-normalizer==3.4.3 # via requests From 0c8e35415d17776d3bd71ab50ae0cc5845a9b932 Mon Sep 17 00:00:00 2001 From: Usama Sadiq Date: Tue, 7 Oct 2025 09:43:50 +0500 Subject: [PATCH 22/54] fix: add Djang<6.0 local constraint --- requirements/constraints.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 292b897f9e..8ae221c9da 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -13,6 +13,10 @@ # This file contains all common constraints for edx-repos -c common_constraints.txt +# Date: 2025-10-07 +# Stay on LTS version, remove once this is added to common constraint +Django<6.0 + # Date: 2020-02-26 # As it is not clarified what exact breaking changes will be introduced as per # the next major release, ensure the installed version is within boundaries. From 9bf7dfa7580c8631aff5af71896906f34f545ec9 Mon Sep 17 00:00:00 2001 From: Usama Sadiq Date: Tue, 7 Oct 2025 09:57:41 +0500 Subject: [PATCH 23/54] chore: python requirements upgrade --- requirements/edx/base.txt | 7 ++++--- requirements/edx/development.txt | 7 ++++--- requirements/edx/doc.txt | 7 ++++--- requirements/edx/testing.txt | 7 ++++--- scripts/user_retirement/requirements/base.txt | 6 ++++-- scripts/user_retirement/requirements/testing.txt | 2 +- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index e1d98bce49..be367e3d89 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -167,8 +167,9 @@ defusedxml==0.7.1 # ora2 # python3-openid # social-auth-core -django==5.2.6 +django==5.2.7 # via + # -c requirements/constraints.txt # -r requirements/edx/kernel.in # django-appconf # django-autocomplete-light @@ -276,7 +277,7 @@ django-fernet-fields-v2==0.9 # via # edx-enterprise # enterprise-integrated-channels -django-filter==25.1 +django-filter==25.2 # via # -r requirements/edx/kernel.in # edx-enterprise @@ -780,7 +781,7 @@ mysqlclient==2.2.7 # via # -r requirements/edx/kernel.in # openedx-forum -nh3==0.3.0 +nh3==0.3.1 # via # -r requirements/edx/kernel.in # xblocks-contrib diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index e13ceca677..136ae43a6f 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -331,8 +331,9 @@ distlib==0.4.0 # via # -r requirements/edx/testing.txt # virtualenv -django==5.2.6 +django==5.2.7 # via + # -c requirements/constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # django-appconf @@ -468,7 +469,7 @@ django-fernet-fields-v2==0.9 # -r requirements/edx/testing.txt # edx-enterprise # enterprise-integrated-channels -django-filter==25.1 +django-filter==25.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1306,7 +1307,7 @@ mysqlclient==2.2.7 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # openedx-forum -nh3==0.3.0 +nh3==0.3.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index f843ead54e..6310ee6cec 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -225,8 +225,9 @@ defusedxml==0.7.1 # ora2 # python3-openid # social-auth-core -django==5.2.6 +django==5.2.7 # via + # -c requirements/constraints.txt # -r requirements/edx/base.txt # django-appconf # django-autocomplete-light @@ -341,7 +342,7 @@ django-fernet-fields-v2==0.9 # -r requirements/edx/base.txt # edx-enterprise # enterprise-integrated-channels -django-filter==25.1 +django-filter==25.2 # via # -r requirements/edx/base.txt # edx-enterprise @@ -949,7 +950,7 @@ mysqlclient==2.2.7 # via # -r requirements/edx/base.txt # openedx-forum -nh3==0.3.0 +nh3==0.3.1 # via # -r requirements/edx/base.txt # xblocks-contrib diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 21946d8e57..895c92da1c 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -251,8 +251,9 @@ dill==0.4.0 # via pylint distlib==0.4.0 # via virtualenv -django==5.2.6 +django==5.2.7 # via + # -c requirements/constraints.txt # -r requirements/edx/base.txt # django-appconf # django-autocomplete-light @@ -367,7 +368,7 @@ django-fernet-fields-v2==0.9 # -r requirements/edx/base.txt # edx-enterprise # enterprise-integrated-channels -django-filter==25.1 +django-filter==25.2 # via # -r requirements/edx/base.txt # edx-enterprise @@ -994,7 +995,7 @@ mysqlclient==2.2.7 # via # -r requirements/edx/base.txt # openedx-forum -nh3==0.3.0 +nh3==0.3.1 # via # -r requirements/edx/base.txt # xblocks-contrib diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index 7aeb62d428..0507348b22 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -31,8 +31,10 @@ click==8.3.0 # -r scripts/user_retirement/requirements/base.in # edx-django-utils cryptography==45.0.7 - # via pyjwt -django==5.2.6 + # via + # -c requirements/constraints.txt + # pyjwt +django==5.2.7 # via # -c requirements/constraints.txt # django-crum diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index b582e229f8..01ba9d65a4 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -52,7 +52,7 @@ cryptography==45.0.7 # pyjwt ddt==1.7.2 # via -r scripts/user_retirement/requirements/testing.in -django==5.2.6 +django==5.2.7 # via # -r scripts/user_retirement/requirements/base.txt # django-crum From 5c759f1e13fd5516bc378cbf36846ad1d3da545e Mon Sep 17 00:00:00 2001 From: Muhammad Farhan Khan Date: Tue, 7 Oct 2025 19:01:50 +0500 Subject: [PATCH 24/54] refactor: Update and migrate Video Block JS files into xmodule/assets - Move Video Block JS files from xmodule/js/src/video/ to xmodule/assets/video/public/js/ - Update JavaScript files from RequireJS to ES6 import/export - test: Enable and fix Karma Js tests for Video XBlock (#37351) --------- Co-authored-by: salmannawaz --- common/static/common/js/karma.common.conf.js | 6 +- package.json | 2 +- webpack.builtinblocks.config.js | 7 +- webpack.common.config.js | 11 +- .../video/public/js/00_async_process.js | 52 + .../assets/video/public/js/00_component.js | 81 + xmodule/assets/video/public/js/00_i18n.js | 35 + xmodule/assets/video/public/js/00_iterator.js | 83 + xmodule/assets/video/public/js/00_resizer.js | 236 +++ xmodule/assets/video/public/js/00_sjson.js | 108 ++ .../video/public/js/00_video_storage.js | 96 ++ .../assets/video/public/js/01_initialize.js | 845 ++++++++++ .../video/public/js/025_focus_grabber.js | 132 ++ .../video/public/js/02_html5_hls_video.js | 145 ++ .../assets/video/public/js/02_html5_video.js | 380 +++++ .../public/js/035_video_accessible_menu.js | 65 + .../public/js/036_video_social_sharing.js | 85 + .../js/037_video_transcript_feedback.js | 240 +++ .../assets/video/public/js/03_video_player.js | 911 ++++++++++ .../video/public/js/04_video_control.js | 164 ++ .../video/public/js/04_video_full_screen.js | 309 ++++ .../public/js/05_video_quality_control.js | 176 ++ .../public/js/06_video_progress_slider.js | 360 ++++ .../public/js/07_video_volume_control.js | 554 +++++++ .../js/08_video_auto_advance_control.js | 134 ++ .../video/public/js/08_video_speed_control.js | 417 +++++ .../video/public/js/095_video_context_menu.js | 698 ++++++++ xmodule/assets/video/public/js/09_bumper.js | 108 ++ .../assets/video/public/js/09_completion.js | 201 +++ .../public/js/09_events_bumper_plugin.js | 112 ++ .../video/public/js/09_events_plugin.js | 177 ++ .../video/public/js/09_play_pause_control.js | 96 ++ .../video/public/js/09_play_placeholder.js | 84 + .../video/public/js/09_play_skip_control.js | 86 + xmodule/assets/video/public/js/09_poster.js | 62 + .../video/public/js/09_save_state_plugin.js | 131 ++ .../assets/video/public/js/09_skip_control.js | 72 + .../video/public/js/09_video_caption.js | 1459 ++++++++++++++++ xmodule/assets/video/public/js/10_commands.js | 108 ++ xmodule/assets/video/public/js/10_main.js | 133 ++ .../video/public/js/utils}/time.js | 12 +- xmodule/js/karma_runner_webpack.js | 13 +- xmodule/js/karma_xmodule_webpack.conf.js | 4 +- xmodule/js/spec/helper.js | 3 +- xmodule/js/spec/time_spec.js | 2 +- xmodule/js/spec/video/async_process_spec.js | 146 +- xmodule/js/spec/video/initialize_spec.js | 598 ++++--- xmodule/js/spec/video/iterator_spec.js | 188 +-- xmodule/js/spec/video/resizer_spec.js | 477 +++--- xmodule/js/spec/video/sjson_spec.js | 108 +- .../js/spec/video/video_autoadvance_spec.js | 7 +- xmodule/js/spec/video/video_caption_spec.js | 16 +- xmodule/js/spec/video/video_control_spec.js | 43 +- .../js/spec/video/video_events_plugin_spec.js | 411 ++--- .../js/spec/video/video_focus_grabber_spec.js | 3 +- xmodule/js/spec/video/video_player_spec.js | 14 +- .../spec/video/video_progress_slider_spec.js | 19 +- .../video/video_save_state_plugin_spec.js | 2 +- .../js/spec/video/video_speed_control_spec.js | 3 +- xmodule/js/spec/video/video_storage_spec.js | 157 +- xmodule/js/src/video/00_async_process.js | 59 - xmodule/js/src/video/00_component.js | 83 - xmodule/js/src/video/00_i18n.js | 40 - xmodule/js/src/video/00_iterator.js | 90 - xmodule/js/src/video/00_resizer.js | 238 --- xmodule/js/src/video/00_sjson.js | 115 -- xmodule/js/src/video/00_video_storage.js | 103 -- xmodule/js/src/video/01_initialize.js | 845 ---------- xmodule/js/src/video/025_focus_grabber.js | 135 -- xmodule/js/src/video/02_html5_hls_video.js | 146 -- xmodule/js/src/video/02_html5_video.js | 390 ----- .../js/src/video/035_video_accessible_menu.js | 70 - .../js/src/video/036_video_social_sharing.js | 92 -- .../video/037_video_transcript_feedback.js | 247 --- xmodule/js/src/video/03_video_player.js | 914 ---------- xmodule/js/src/video/04_video_control.js | 169 -- xmodule/js/src/video/04_video_full_screen.js | 313 ---- .../js/src/video/05_video_quality_control.js | 181 -- .../js/src/video/06_video_progress_slider.js | 367 ----- .../js/src/video/07_video_volume_control.js | 554 ------- .../js/src/video/08_video_speed_control.js | 422 ----- .../js/src/video/095_video_context_menu.js | 687 -------- xmodule/js/src/video/09_bumper.js | 112 -- xmodule/js/src/video/09_completion.js | 202 --- .../js/src/video/09_events_bumper_plugin.js | 113 -- xmodule/js/src/video/09_events_plugin.js | 179 -- xmodule/js/src/video/09_play_pause_control.js | 97 -- xmodule/js/src/video/09_play_placeholder.js | 88 - xmodule/js/src/video/09_play_skip_control.js | 90 - xmodule/js/src/video/09_poster.js | 70 - xmodule/js/src/video/09_save_state_plugin.js | 131 -- xmodule/js/src/video/09_skip_control.js | 76 - xmodule/js/src/video/09_video_caption.js | 1463 ----------------- xmodule/js/src/video/10_commands.js | 111 -- xmodule/js/src/video/10_main.js | 198 --- xmodule/video_block/video_block.py | 6 +- 96 files changed, 10262 insertions(+), 10321 deletions(-) create mode 100644 xmodule/assets/video/public/js/00_async_process.js create mode 100644 xmodule/assets/video/public/js/00_component.js create mode 100644 xmodule/assets/video/public/js/00_i18n.js create mode 100644 xmodule/assets/video/public/js/00_iterator.js create mode 100644 xmodule/assets/video/public/js/00_resizer.js create mode 100644 xmodule/assets/video/public/js/00_sjson.js create mode 100644 xmodule/assets/video/public/js/00_video_storage.js create mode 100644 xmodule/assets/video/public/js/01_initialize.js create mode 100644 xmodule/assets/video/public/js/025_focus_grabber.js create mode 100644 xmodule/assets/video/public/js/02_html5_hls_video.js create mode 100644 xmodule/assets/video/public/js/02_html5_video.js create mode 100644 xmodule/assets/video/public/js/035_video_accessible_menu.js create mode 100644 xmodule/assets/video/public/js/036_video_social_sharing.js create mode 100644 xmodule/assets/video/public/js/037_video_transcript_feedback.js create mode 100644 xmodule/assets/video/public/js/03_video_player.js create mode 100644 xmodule/assets/video/public/js/04_video_control.js create mode 100644 xmodule/assets/video/public/js/04_video_full_screen.js create mode 100644 xmodule/assets/video/public/js/05_video_quality_control.js create mode 100644 xmodule/assets/video/public/js/06_video_progress_slider.js create mode 100644 xmodule/assets/video/public/js/07_video_volume_control.js create mode 100644 xmodule/assets/video/public/js/08_video_auto_advance_control.js create mode 100644 xmodule/assets/video/public/js/08_video_speed_control.js create mode 100644 xmodule/assets/video/public/js/095_video_context_menu.js create mode 100644 xmodule/assets/video/public/js/09_bumper.js create mode 100644 xmodule/assets/video/public/js/09_completion.js create mode 100644 xmodule/assets/video/public/js/09_events_bumper_plugin.js create mode 100644 xmodule/assets/video/public/js/09_events_plugin.js create mode 100644 xmodule/assets/video/public/js/09_play_pause_control.js create mode 100644 xmodule/assets/video/public/js/09_play_placeholder.js create mode 100644 xmodule/assets/video/public/js/09_play_skip_control.js create mode 100644 xmodule/assets/video/public/js/09_poster.js create mode 100644 xmodule/assets/video/public/js/09_save_state_plugin.js create mode 100644 xmodule/assets/video/public/js/09_skip_control.js create mode 100644 xmodule/assets/video/public/js/09_video_caption.js create mode 100644 xmodule/assets/video/public/js/10_commands.js create mode 100644 xmodule/assets/video/public/js/10_main.js rename xmodule/{js/src => assets/video/public/js/utils}/time.js (80%) delete mode 100644 xmodule/js/src/video/00_async_process.js delete mode 100644 xmodule/js/src/video/00_component.js delete mode 100644 xmodule/js/src/video/00_i18n.js delete mode 100644 xmodule/js/src/video/00_iterator.js delete mode 100644 xmodule/js/src/video/00_resizer.js delete mode 100644 xmodule/js/src/video/00_sjson.js delete mode 100644 xmodule/js/src/video/00_video_storage.js delete mode 100644 xmodule/js/src/video/01_initialize.js delete mode 100644 xmodule/js/src/video/025_focus_grabber.js delete mode 100644 xmodule/js/src/video/02_html5_hls_video.js delete mode 100644 xmodule/js/src/video/02_html5_video.js delete mode 100644 xmodule/js/src/video/035_video_accessible_menu.js delete mode 100644 xmodule/js/src/video/036_video_social_sharing.js delete mode 100644 xmodule/js/src/video/037_video_transcript_feedback.js delete mode 100644 xmodule/js/src/video/03_video_player.js delete mode 100644 xmodule/js/src/video/04_video_control.js delete mode 100644 xmodule/js/src/video/04_video_full_screen.js delete mode 100644 xmodule/js/src/video/05_video_quality_control.js delete mode 100644 xmodule/js/src/video/06_video_progress_slider.js delete mode 100644 xmodule/js/src/video/07_video_volume_control.js delete mode 100644 xmodule/js/src/video/08_video_speed_control.js delete mode 100644 xmodule/js/src/video/095_video_context_menu.js delete mode 100644 xmodule/js/src/video/09_bumper.js delete mode 100644 xmodule/js/src/video/09_completion.js delete mode 100644 xmodule/js/src/video/09_events_bumper_plugin.js delete mode 100644 xmodule/js/src/video/09_events_plugin.js delete mode 100644 xmodule/js/src/video/09_play_pause_control.js delete mode 100644 xmodule/js/src/video/09_play_placeholder.js delete mode 100644 xmodule/js/src/video/09_play_skip_control.js delete mode 100644 xmodule/js/src/video/09_poster.js delete mode 100644 xmodule/js/src/video/09_save_state_plugin.js delete mode 100644 xmodule/js/src/video/09_skip_control.js delete mode 100644 xmodule/js/src/video/09_video_caption.js delete mode 100644 xmodule/js/src/video/10_commands.js delete mode 100644 xmodule/js/src/video/10_main.js diff --git a/common/static/common/js/karma.common.conf.js b/common/static/common/js/karma.common.conf.js index 16501f47ad..b49db1298d 100644 --- a/common/static/common/js/karma.common.conf.js +++ b/common/static/common/js/karma.common.conf.js @@ -332,7 +332,11 @@ function getBaseConfig(config, useRequireJs) { base: 'Firefox', prefs: { 'app.update.auto': false, - 'app.update.enabled': false + 'app.update.enabled': false, + 'media.autoplay.default': 0, // allow autoplay + 'media.autoplay.blocking_policy': 0, // disable autoplay blocking + 'media.autoplay.allow-extension-background-pages': true, + 'media.autoplay.enabled.user-gestures-needed': false, } }, ChromeDocker: { diff --git a/package.json b/package.json index a8b762450e..d3e14e5360 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "watch-sass": "scripts/watch_sass.sh", "test": "npm run test-jest && npm run test-karma", "test-jest": "jest", - "test-karma": "npm run test-karma-vanilla && npm run test-karma-require && echo 'WARNING: Skipped broken webpack tests. For details, see: https://github.com/openedx/edx-platform/issues/35956'", + "test-karma": "npm run test-karma-vanilla && npm run test-karma-require && npm run test-xmodule-webpack && echo 'WARNING: Skipped broken lms-webpack and cms-webpack tests. For details, see: https://github.com/openedx/edx-platform/issues/35956'", "test-karma-vanilla": "npm run test-cms-vanilla && npm run test-xmodule-vanilla && npm run test-common-vanilla", "test-karma-require": "npm run test-cms-require && npm run test-common-require", "test-karma-webpack": "npm run test-cms-webpack && npm run test-lms-webpack && npm run test-xmodule-webpack", diff --git a/webpack.builtinblocks.config.js b/webpack.builtinblocks.config.js index 1c5a9b1e0e..c0f2fdaeb4 100644 --- a/webpack.builtinblocks.config.js +++ b/webpack.builtinblocks.config.js @@ -79,14 +79,13 @@ module.exports = { './xmodule/js/src/xmodule.js', './xmodule/js/src/sequence/edit.js' ], - VideoBlockDisplay: [ - './xmodule/js/src/xmodule.js', - './xmodule/js/src/video/10_main.js' - ], VideoBlockEditor: [ './xmodule/js/src/xmodule.js', './xmodule/js/src/tabs/tabs-aggregator.js' ], + VideoBlockDisplay: [ + './xmodule/assets/video/public/js/10_main.js' + ], WordCloudBlockDisplay: [ './xmodule/js/src/xmodule.js', './xmodule/assets/word_cloud/src/js/word_cloud.js' diff --git a/webpack.common.config.js b/webpack.common.config.js index 36ac75708c..f6fd4fa357 100644 --- a/webpack.common.config.js +++ b/webpack.common.config.js @@ -505,15 +505,6 @@ module.exports = Merge.merge({ } ] }, - { - test: /xmodule\/js\/src\/video\/10_main.js/, - use: [ - { - loader: 'imports-loader', - options: 'this=>window' - } - ] - }, /* * END BUILT-IN XBLOCK ASSETS WITH GLOBAL DEFINITIONS ***************************************************************************************************** */ @@ -680,9 +671,11 @@ module.exports = Merge.merge({ $: 'jQuery', backbone: 'Backbone', canvas: 'canvas', + fs: 'fs', gettext: 'gettext', jquery: 'jQuery', logger: 'Logger', + path: 'path', underscore: '_', URI: 'URI', XBlockToXModuleShim: 'XBlockToXModuleShim', diff --git a/xmodule/assets/video/public/js/00_async_process.js b/xmodule/assets/video/public/js/00_async_process.js new file mode 100644 index 0000000000..a909e8225a --- /dev/null +++ b/xmodule/assets/video/public/js/00_async_process.js @@ -0,0 +1,52 @@ +'use strict'; + +/** + * Provides convenient way to process big amount of data without UI blocking. + * + * @param {array} list Array to process. + * @param {function} process Calls this function on each item in the list. + * @return {array} Returns a Promise object to observe when all actions of a + * certain type bound to the collection, queued or not, have finished. + */ +let AsyncProcess = { + array: function(list, process) { + if (!_.isArray(list)) { + return $.Deferred().reject().promise(); + } + + if (!_.isFunction(process) || !list.length) { + return $.Deferred().resolve(list).promise(); + } + + let MAX_DELAY = 50, // maximum amount of time that js code should be allowed to run continuously + dfd = $.Deferred(); + let result = []; + let index = 0; + let len = list.length; + + let getCurrentTime = function() { + return (new Date()).getTime(); + }; + + let handler = function() { + let start = getCurrentTime(); + + do { + result[index] = process(list[index], index); + index++; + } while (index < len && getCurrentTime() - start < MAX_DELAY); + + if (index < len) { + setTimeout(handler, 25); + } else { + dfd.resolve(result); + } + }; + + setTimeout(handler, 25); + + return dfd.promise(); + } +}; + +export default AsyncProcess; diff --git a/xmodule/assets/video/public/js/00_component.js b/xmodule/assets/video/public/js/00_component.js new file mode 100644 index 0000000000..2ac183b198 --- /dev/null +++ b/xmodule/assets/video/public/js/00_component.js @@ -0,0 +1,81 @@ +'use strict'; + +import _ from 'underscore'; + + +/** + * Creates a new object with the specified prototype object and properties. + * @param {Object} o The object which should be the prototype of the + * newly-created object. + * @private + * @throws {TypeError, Error} + * @return {Object} + */ +let inherit = Object.create || (function() { + let F = function() {}; + + return function(o) { + if (arguments.length > 1) { + throw Error('Second argument not supported'); + } + if (_.isNull(o) || _.isUndefined(o)) { + throw Error('Cannot set a null [[Prototype]]'); + } + if (!_.isObject(o)) { + throw TypeError('Argument must be an object'); + } + + F.prototype = o; + + return new F(); + }; +}()); + +/** + * Component module. + * @exports video/00_component.js + * @constructor + * @return {jquery Promise} + */ +let Component = function() { + if ($.isFunction(this.initialize)) { + // eslint-disable-next-line prefer-spread + return this.initialize.apply(this, arguments); + } +}; + +/** + * Returns new constructor that inherits form the current constructor. + * @static + * @param {Object} protoProps The object containing which will be added to + * the prototype. + * @return {Object} + */ +Component.extend = function(protoProps, staticProps) { + let Parent = this; + let Child = function() { + if ($.isFunction(this.initialize)) { + // eslint-disable-next-line prefer-spread + return this.initialize.apply(this, arguments); + } + }; + + // Inherit methods and properties from the Parent prototype. + Child.prototype = inherit(Parent.prototype); + Child.constructor = Parent; + // Provide access to parent's methods and properties + Child.__super__ = Parent.prototype; + + // Extends inherited methods and properties by methods/properties + // passed as argument. + if (protoProps) { + $.extend(Child.prototype, protoProps); + } + + // Inherit static methods and properties + $.extend(Child, Parent, staticProps); + + return Child; +}; + +export default Component; diff --git a/xmodule/assets/video/public/js/00_i18n.js b/xmodule/assets/video/public/js/00_i18n.js new file mode 100644 index 0000000000..1962ed4ee8 --- /dev/null +++ b/xmodule/assets/video/public/js/00_i18n.js @@ -0,0 +1,35 @@ +'use strict'; + +/** + * i18n module. + * @exports video/00_i18n.js + * @return {object} + */ + +let i18n = { + Play: gettext('Play'), + Pause: gettext('Pause'), + Mute: gettext('Mute'), + Unmute: gettext('Unmute'), + 'Exit full browser': gettext('Exit full browser'), + 'Fill browser': gettext('Fill browser'), + Speed: gettext('Speed'), + 'Auto-advance': gettext('Auto-advance'), + Volume: gettext('Volume'), + // Translators: Volume level equals 0%. + Muted: gettext('Muted'), + // Translators: Volume level in range ]0,20]% + 'Very low': gettext('Very low'), + // Translators: Volume level in range ]20,40]% + Low: gettext('Low'), + // Translators: Volume level in range ]40,60]% + Average: gettext('Average'), + // Translators: Volume level in range ]60,80]% + Loud: gettext('Loud'), + // Translators: Volume level in range ]80,99]% + 'Very loud': gettext('Very loud'), + // Translators: Volume level equals 100%. + Maximum: gettext('Maximum') +}; + +export default i18n; diff --git a/xmodule/assets/video/public/js/00_iterator.js b/xmodule/assets/video/public/js/00_iterator.js new file mode 100644 index 0000000000..5b597f200e --- /dev/null +++ b/xmodule/assets/video/public/js/00_iterator.js @@ -0,0 +1,83 @@ +'use strict'; + +/** + * Provides convenient way to work with iterable data. + * @exports video/00_iterator.js + * @constructor + * @param {array} list Array to be iterated. + */ +let Iterator = function(list) { + this.list = list; + this.index = 0; + this.size = this.list.length; + this.lastIndex = this.list.length - 1; +}; + +Iterator.prototype = { + + /** + * Checks validity of provided index for the iterator. + * @access protected + * @param {numebr} index + * @return {boolean} + */ + _isValid: function(index) { + return _.isNumber(index) && index < this.size && index >= 0; + }, + + /** + * Returns next element. + * @param {number} [index] Updates current position. + * @return {any} + */ + next: function(index) { + if (!(this._isValid(index))) { + index = this.index; + } + + this.index = (index >= this.lastIndex) ? 0 : index + 1; + + return this.list[this.index]; + }, + + /** + * Returns previous element. + * @param {number} [index] Updates current position. + * @return {any} + */ + prev: function(index) { + if (!(this._isValid(index))) { + index = this.index; + } + + this.index = (index < 1) ? this.lastIndex : index - 1; + + return this.list[this.index]; + }, + + /** + * Returns last element in the list. + * @return {any} + */ + last: function() { + return this.list[this.lastIndex]; + }, + + /** + * Returns first element in the list. + * @return {any} + */ + first: function() { + return this.list[0]; + }, + + /** + * Returns `true` if current position is last for the iterator. + * @return {boolean} + */ + isEnd: function() { + return this.index === this.lastIndex; + } +}; + +export default Iterator; diff --git a/xmodule/assets/video/public/js/00_resizer.js b/xmodule/assets/video/public/js/00_resizer.js new file mode 100644 index 0000000000..d892ec4d18 --- /dev/null +++ b/xmodule/assets/video/public/js/00_resizer.js @@ -0,0 +1,236 @@ +'use strict'; + +import _ from 'underscore'; + + +let Resizer = function(params) { + let defaults = { + container: window, + element: null, + containerRatio: null, + elementRatio: null + }, + callbacksList = [], + delta = { + height: 0, + width: 0 + }, + module = {}; + let mode = null, + config; + + // eslint-disable-next-line no-shadow + let initialize = function(params) { + if (!config) { + config = defaults; + } + + config = $.extend(true, {}, config, params); + + if (!config.element) { + console.log( + 'Required parameter `element` is not passed.' + ); + } + + return module; + }; + + let getData = function() { + let $container = $(config.container), + containerWidth = $container.width() + delta.width, + containerHeight = $container.height() + delta.height; + let containerRatio = config.containerRatio; + + let $element = $(config.element); + let elementRatio = config.elementRatio; + + if (!containerRatio) { + containerRatio = containerWidth / containerHeight; + } + + if (!elementRatio) { + elementRatio = $element.width() / $element.height(); + } + + return { + containerWidth: containerWidth, + containerHeight: containerHeight, + containerRatio: containerRatio, + element: $element, + elementRatio: elementRatio + }; + }; + + let align = function() { + let data = getData(); + + switch (mode) { + case 'height': + alignByHeightOnly(); + break; + + case 'width': + alignByWidthOnly(); + break; + + default: + if (data.containerRatio >= data.elementRatio) { + alignByHeightOnly(); + } else { + alignByWidthOnly(); + } + break; + } + + fireCallbacks(); + + return module; + }; + + let alignByWidthOnly = function() { + let data = getData(), + height = data.containerWidth / data.elementRatio; + + data.element.css({ + height: height, + width: data.containerWidth, + top: 0.5 * (data.containerHeight - height), + left: 0 + }); + + return module; + }; + + let alignByHeightOnly = function() { + let data = getData(), + width = data.containerHeight * data.elementRatio; + + data.element.css({ + height: data.containerHeight, + width: data.containerHeight * data.elementRatio, + top: 0, + left: 0.5 * (data.containerWidth - width) + }); + + return module; + }; + + let setMode = function(param) { + if (_.isString(param)) { + mode = param; + align(); + } + + return module; + }; + + let setElement = function(element) { + config.element = element; + + return module; + }; + + let addCallback = function(func) { + if ($.isFunction(func)) { + callbacksList.push(func); + } else { + console.error('[Video info]: TypeError: Argument is not a function.'); + } + + return module; + }; + + let addOnceCallback = function(func) { + if ($.isFunction(func)) { + let decorator = function() { + func(); + removeCallback(func); + }; + + addCallback(decorator); + } else { + console.error('TypeError: Argument is not a function.'); + } + + return module; + }; + + let fireCallbacks = function() { + $.each(callbacksList, function(index, callback) { + callback(); + }); + }; + + let removeCallbacks = function() { + callbacksList.length = 0; + + return module; + }; + + let removeCallback = function(func) { + let index = $.inArray(func, callbacksList); + + if (index !== -1) { + return callbacksList.splice(index, 1); + } + }; + + let resetDelta = function() { + // eslint-disable-next-line no-multi-assign + delta.height = delta.width = 0; + + return module; + }; + + let addDelta = function(value, side) { + if (_.isNumber(value) && _.isNumber(delta[side])) { + delta[side] += value; + } + + return module; + }; + + let substractDelta = function(value, side) { + if (_.isNumber(value) && _.isNumber(delta[side])) { + delta[side] -= value; + } + + return module; + }; + + let destroy = function() { + let data = getData(); + data.element.css({ + height: '', width: '', top: '', left: '' + }); + removeCallbacks(); + resetDelta(); + mode = null; + }; + + initialize.apply(module, arguments); + + return $.extend(true, module, { + align: align, + alignByWidthOnly: alignByWidthOnly, + alignByHeightOnly: alignByHeightOnly, + destroy: destroy, + setParams: initialize, + setMode: setMode, + setElement: setElement, + callbacks: { + add: addCallback, + once: addOnceCallback, + remove: removeCallback, + removeAll: removeCallbacks + }, + delta: { + add: addDelta, + substract: substractDelta, + reset: resetDelta + } + }); +}; + +export default Resizer; diff --git a/xmodule/assets/video/public/js/00_sjson.js b/xmodule/assets/video/public/js/00_sjson.js new file mode 100644 index 0000000000..99d870ff84 --- /dev/null +++ b/xmodule/assets/video/public/js/00_sjson.js @@ -0,0 +1,108 @@ +'use strict'; + +let Sjson = function(data) { + let sjson = { + start: data.start.concat(), + text: data.text.concat() + }, + module = {}; + + let getter = function(propertyName) { + return function() { + return sjson[propertyName]; + }; + }; + + let getStartTimes = getter('start'); + + let getCaptions = getter('text'); + + let size = function() { + return sjson.text.length; + }; + + function search(time, startTime, endTime) { + let start = getStartTimes(), + max = size() - 1, + min = 0, + results, + index; + + // if we specify a start and end time to search, + // search the filtered list of captions in between + // the start / end times. + // Else, search the unfiltered list. + if (typeof startTime !== 'undefined' + && typeof endTime !== 'undefined') { + results = filter(startTime, endTime); + start = results.start; + max = results.captions.length - 1; + } else { + start = getStartTimes(); + } + while (min < max) { + index = Math.ceil((max + min) / 2); + + if (time < start[index]) { + max = index - 1; + } + + if (time >= start[index]) { + min = index; + } + } + + return min; + } + + function filter(start, end) { + /* filters captions that occur between inputs + * `start` and `end`. Start and end should + * be Numbers (doubles) corresponding to the + * number of seconds elapsed since the beginning + * of the video. + * + * Returns an object with properties + * "start" and "captions" representing + * parallel arrays of start times and + * their corresponding captions. + */ + let filteredTimes = []; + let filteredCaptions = []; + let startTimes = getStartTimes(); + let captions = getCaptions(); + + if (startTimes.length !== captions.length) { + console.warn('video caption and start time arrays do not match in length'); + } + + // if end is null, then it's been set to + // some erroneous value, so filter using the + // entire array as long as it's not empty + if (end === null && startTimes.length) { + end = startTimes[startTimes.length - 1]; + } + + _.filter(startTimes, function(currentStartTime, i) { + if (currentStartTime >= start && currentStartTime <= end) { + filteredTimes.push(currentStartTime); + filteredCaptions.push(captions[i]); + } + }); + + return { + start: filteredTimes, + captions: filteredCaptions + }; + } + + return { + getCaptions: getCaptions, + getStartTimes: getStartTimes, + getSize: size, + filter: filter, + search: search + }; +}; + +export default Sjson; diff --git a/xmodule/assets/video/public/js/00_video_storage.js b/xmodule/assets/video/public/js/00_video_storage.js new file mode 100644 index 0000000000..f2293336fe --- /dev/null +++ b/xmodule/assets/video/public/js/00_video_storage.js @@ -0,0 +1,96 @@ +'use strict'; + +/** + * Provides convenient way to store key value pairs. + * + * @param {string} namespace Namespace that is used to store data. + * @return {object} VideoStorage API. + */ +let VideoStorage = function(namespace, id) { + /** + * Adds new value to the storage or rewrites existent. + * + * @param {string} name Identifier of the data. + * @param {any} value Data to store. + * @param {boolean} instanceSpecific Data with this flag will be added + * to instance specific storage. + */ + let setItem = function(name, value, instanceSpecific) { + if (name) { + if (instanceSpecific) { + window[namespace][id][name] = value; + } else { + window[namespace][name] = value; + } + } + }; + + /** + * Returns the current value associated with the given name. + * + * @param {string} name Identifier of the data. + * @param {boolean} instanceSpecific Data with this flag will be added + * to instance specific storage. + * @return {any} The current value associated with the given name. + * If the given key does not exist in the list + * associated with the object then this method must return null. + */ + let getItem = function(name, instanceSpecific) { + if (instanceSpecific) { + return window[namespace][id][name]; + } else { + return window[namespace][name]; + } + }; + + /** + * Removes the current value associated with the given name. + * + * @param {string} name Identifier of the data. + * @param {boolean} instanceSpecific Data with this flag will be added + * to instance specific storage. + */ + let removeItem = function(name, instanceSpecific) { + if (instanceSpecific) { + delete window[namespace][id][name]; + } else { + delete window[namespace][name]; + } + }; + + /** + * Empties the storage. + * + */ + let clear = function() { + window[namespace] = {}; + window[namespace][id] = {}; + }; + + /** + * Initializes the module: creates a storage with proper namespace. + * + * @private + */ + (function initialize() { + if (!namespace) { + namespace = 'VideoStorage'; + } + if (!id) { + // Generate random alpha-numeric string. + id = Math.random().toString(36).slice(2); + } + + window[namespace] = window[namespace] || {}; + window[namespace][id] = window[namespace][id] || {}; + }()); + + return { + clear: clear, + getItem: getItem, + removeItem: removeItem, + setItem: setItem + }; +}; + +export default VideoStorage; diff --git a/xmodule/assets/video/public/js/01_initialize.js b/xmodule/assets/video/public/js/01_initialize.js new file mode 100644 index 0000000000..85248b3f02 --- /dev/null +++ b/xmodule/assets/video/public/js/01_initialize.js @@ -0,0 +1,845 @@ +/* eslint-disable no-console, no-param-reassign */ +/** + * @file Initialize module works with the JSON config, and sets up various + * settings, parameters, variables. After all setup actions are performed, it + * invokes the video player to play the specified video. This module must be + * invoked first. It provides several functions which do not fit in with other + * modules. + * + * @external VideoPlayer + * + * @module Initialize + */ + +import VideoPlayer from './03_video_player.js'; +import i18n from './00_i18n.js'; +import _ from 'underscore'; +import moment from 'moment'; + +/** + * @function + * + * Initialize module exports this function. + * + * @param {object} state The object containg the state of the video player. + * All other modules, their parameters, public variables, etc. are + * available via this object. + * @param {DOM element} element Container of the entire Video DOM element. + */ +let Initialize = function(state, element) { + _makeFunctionsPublic(state); + + state.initialize(element) + .done(function() { + if (state.isYoutubeType()) { + state.parseSpeed(); + } + // On iPhones and iPods native controls are used. + if (/iP(hone|od)/i.test(state.isTouch[0])) { + _hideWaitPlaceholder(state); + state.el.trigger('initialize', arguments); + + return false; + } + + _initializeModules(state, i18n) + .done(function() { + // On iPad ready state occurs just after start playing. + // We hide controls before video starts playing. + if (/iPad|Android/i.test(state.isTouch[0])) { + state.el.on('play', _.once(function() { + state.trigger('videoControl.show', null); + })); + } else { + // On PC show controls immediately. + state.trigger('videoControl.show', null); + } + + _hideWaitPlaceholder(state); + state.el.trigger('initialize', arguments); + }); + }); +}; + +/* eslint-disable no-use-before-define */ +let methodsDict = { + bindTo: bindTo, + fetchMetadata: fetchMetadata, + getCurrentLanguage: getCurrentLanguage, + getDuration: getDuration, + getPlayerMode: getPlayerMode, + getVideoMetadata: getVideoMetadata, + initialize: initialize, + isHtml5Mode: isHtml5Mode, + isFlashMode: isFlashMode, + isYoutubeType: isYoutubeType, + parseSpeed: parseSpeed, + parseYoutubeStreams: parseYoutubeStreams, + setPlayerMode: setPlayerMode, + setSpeed: setSpeed, + setAutoAdvance: setAutoAdvance, + speedToString: speedToString, + trigger: trigger, + youtubeId: youtubeId, + loadHtmlPlayer: loadHtmlPlayer, + loadYoutubePlayer: loadYoutubePlayer, + loadYouTubeIFrameAPI: loadYouTubeIFrameAPI +}; +/* eslint-enable no-use-before-define */ + +let _youtubeApiDeferred = null; +let _oldOnYouTubeIframeAPIReady; + +Initialize.prototype = methodsDict; + +export default Initialize; + +// *************************************************************** +// Private functions start here. Private functions start with underscore. +// *************************************************************** + +/** + * @function _makeFunctionsPublic + * + * Functions which will be accessible via 'state' object. When called, + * these functions will get the 'state' + * object as a context. + * + * @param {object} state The object containg the state (properties, + * methods, modules) of the Video player. + */ +function _makeFunctionsPublic(state) { + bindTo(methodsDict, state, state); +} + +// function _renderElements(state) +// +// Create any necessary DOM elements, attach them, and set their +// initial configuration. Also make the created DOM elements available +// via the 'state' object. Much easier to work this way - you don't +// have to do repeated jQuery element selects. +function _renderElements(state) { + // Launch embedding of actual video content, or set it up so that it + // will be done as soon as the appropriate video player (YouTube or + // stand-alone HTML5) is loaded, and can handle embedding. + // + // Note that the loading of stand alone HTML5 player API is handled by + // Require JS. At the time when we reach this code, the stand alone + // HTML5 player is already loaded, so no further testing in that case + // is required. + let video; + let onYTApiReady; + let setupOnYouTubeIframeAPIReady; + + if (state.videoType === 'youtube') { + state.youtubeApiAvailable = false; + + onYTApiReady = function() { + console.log('[Video info]: YouTube API is available and is loaded.'); + if (state.htmlPlayerLoaded) { return; } + + console.log('[Video info]: Starting YouTube player.'); + video = VideoPlayer(state); + + state.modules.push(video); + state.__dfd__.resolve(); + state.youtubeApiAvailable = true; + }; + + if (window.YT) { + // If we have a Deferred object responsible for calling OnYouTubeIframeAPIReady + // callbacks, make sure that they have all been called by trying to resolve the + // Deferred object. Upon resolving, all the OnYouTubeIframeAPIReady will be + // called. If the object has been already resolved, the callbacks will not + // be called a second time. + if (_youtubeApiDeferred) { + _youtubeApiDeferred.resolve(); + } + + window.YT.ready(onYTApiReady); + } else { + // There is only one global variable window.onYouTubeIframeAPIReady which + // is supposed to be a function that will be called by the YouTube API + // when it finished initializing. This function will update this global function + // so that it resolves our Deferred object, which will call all of the + // OnYouTubeIframeAPIReady callbacks. + // + // If this global function is already defined, we store it first, and make + // sure that it gets executed when our Deferred object is resolved. + setupOnYouTubeIframeAPIReady = function() { + _oldOnYouTubeIframeAPIReady = window.onYouTubeIframeAPIReady || undefined; + + window.onYouTubeIframeAPIReady = function() { + _youtubeApiDeferred.resolve(); + }; + + window.onYouTubeIframeAPIReady.done = _youtubeApiDeferred.done; + + if (_oldOnYouTubeIframeAPIReady) { + window.onYouTubeIframeAPIReady.done(_oldOnYouTubeIframeAPIReady); + } + }; + + // If a Deferred object hasn't been created yet, create one now. It will + // be responsible for calling OnYouTubeIframeAPIReady callbacks once the + // YouTube API loads. After creating the Deferred object, load the YouTube + // API. + if (!_youtubeApiDeferred) { + _youtubeApiDeferred = $.Deferred(); + setupOnYouTubeIframeAPIReady(); + } else if (!window.onYouTubeIframeAPIReady || !window.onYouTubeIframeAPIReady.done) { + // The Deferred object could have been already defined in a previous + // initialization of the video module. However, since then the global variable + // window.onYouTubeIframeAPIReady could have been overwritten. If so, + // we should set it up again. + setupOnYouTubeIframeAPIReady(); + } + + // Attach a callback to our Deferred object to be called once the + // YouTube API loads. + window.onYouTubeIframeAPIReady.done(function() { + window.YT.ready(onYTApiReady); + }); + } + } else { + video = VideoPlayer(state); + + state.modules.push(video); + state.__dfd__.resolve(); + state.htmlPlayerLoaded = true; + } +} + +function _waitForYoutubeApi(state) { + console.log('[Video info]: Starting to wait for YouTube API to load.'); + window.setTimeout(function() { + // If YouTube API will load OK, it will run `onYouTubeIframeAPIReady` + // callback, which will set `state.youtubeApiAvailable` to `true`. + // If something goes wrong at this stage, `state.youtubeApiAvailable` is + // `false`. + if (!state.youtubeApiAvailable) { + console.log('[Video info]: YouTube API is not available.'); + if (!state.htmlPlayerLoaded) { + state.loadHtmlPlayer(); + } + } + state.el.trigger('youtube_availability', [state.youtubeApiAvailable]); + }, state.config.ytTestTimeout); +} + +function loadYouTubeIFrameAPI(scriptTag) { + let firstScriptTag = document.getElementsByTagName('script')[0]; + firstScriptTag.parentNode.insertBefore(scriptTag, firstScriptTag); +} + +// function _parseYouTubeIDs(state) +// The function parse YouTube stream ID's. +// @return +// false: We don't have YouTube video IDs to work with; most likely +// we have HTML5 video sources. +// true: Parsing of YouTube video IDs went OK, and we can proceed +// onwards to play YouTube videos. +function _parseYouTubeIDs(state) { + if (state.parseYoutubeStreams(state.config.streams)) { + state.videoType = 'youtube'; + + return true; + } + + console.log( + '[Video info]: Youtube Video IDs are incorrect or absent.' + ); + + return false; +} + +/** + * Extract HLS video URLs from available video URLs. + * + * @param {object} state The object contaning the state (properties, methods, modules) of the Video player. + * @returns Array of available HLS video source urls. + */ +function extractHLSVideoSources(state) { + return _.filter(state.config.sources, function(source) { + return /\.m3u8(\?.*)?$/.test(source); + }); +} + +// function _prepareHTML5Video(state) +// The function prepare HTML5 video, parse HTML5 +// video sources etc. +function _prepareHTML5Video(state) { + state.speeds = ['0.75', '1.0', '1.25', '1.50', '2.0']; + // If none of the supported video formats can be played and there is no + // short-hand video links, than hide the spinner and show error message. + if (!state.config.sources.length) { + _hideWaitPlaceholder(state); + state.el + .find('.video-player div') + .addClass('hidden'); + state.el + .find('.video-player .video-error') + .removeClass('is-hidden'); + + return false; + } + + state.videoType = 'html5'; + + if (!_.keys(state.config.transcriptLanguages).length) { + state.config.showCaptions = false; + } + state.setSpeed(state.speed); + + return true; +} + +function _hideWaitPlaceholder(state) { + state.el + .addClass('is-initialized') + .find('.spinner') + .attr({ + 'aria-hidden': 'true', + tabindex: -1 + }); +} + +function _setConfigurations(state) { + state.setPlayerMode(state.config.mode); + // Possible value are: 'visible', 'hiding', and 'invisible'. + state.controlState = 'visible'; + state.controlHideTimeout = null; + state.captionState = 'invisible'; + state.captionHideTimeout = null; + state.HLSVideoSources = extractHLSVideoSources(state); +} + +// eslint-disable-next-line no-shadow +function _initializeModules(state, i18n) { + let dfd = $.Deferred(), + modulesList = $.map(state.modules, function(module) { + let options = state.options[module.moduleName] || {}; + if (_.isFunction(module)) { + return module(state, i18n, options); + } else if ($.isPlainObject(module)) { + return module; + } + }); + + $.when.apply(null, modulesList) + .done(dfd.resolve); + + return dfd.promise(); +} + +function _getConfiguration(data, storage) { + let isBoolean = function(value) { + let regExp = /^true$/i; + return regExp.test(value.toString()); + }, + // List of keys that will be extracted form the configuration. + extractKeys = [], + // Compatibility keys used to change names of some parameters in + // the final configuration. + compatKeys = { + start: 'startTime', + end: 'endTime' + }, + // Conversions used to pre-process some configuration data. + conversions = { + showCaptions: isBoolean, + autoplay: isBoolean, + autohideHtml5: isBoolean, + autoAdvance: function(value) { + let shouldAutoAdvance = storage.getItem('auto_advance'); + if (_.isUndefined(shouldAutoAdvance)) { + return isBoolean(value) || false; + } else { + return shouldAutoAdvance; + } + }, + savedVideoPosition: function(value) { + return storage.getItem('savedVideoPosition', true) + || Number(value) + || 0; + }, + speed: function(value) { + return storage.getItem('speed', true) || value; + }, + generalSpeed: function(value) { + return storage.getItem('general_speed') + || value + || '1.0'; + }, + transcriptLanguage: function(value) { + return storage.getItem('language') + || value + || 'en'; + }, + ytTestTimeout: function(value) { + value = parseInt(value, 10); + + if (!isFinite(value)) { + value = 1500; + } + + return value; + }, + startTime: function(value) { + value = parseInt(value, 10); + if (!isFinite(value) || value < 0) { + return 0; + } + + return value; + }, + endTime: function(value) { + value = parseInt(value, 10); + + if (!isFinite(value) || value === 0) { + return null; + } + + return value; + } + }, + config = {}; + + data = _.extend({ + startTime: 0, + endTime: null, + sub: '', + streams: '' + }, data); + + $.each(data, function(option, value) { + // Extract option that is in `extractKeys`. + if ($.inArray(option, extractKeys) !== -1) { + return; + } + + // Change option name to key that is in `compatKeys`. + if (compatKeys[option]) { + option = compatKeys[option]; + } + + // Pre-process data. + if (conversions[option]) { + if (_.isFunction(conversions[option])) { + value = conversions[option].call(this, value); + } else { + throw new TypeError(option + ' is not a function.'); + } + } + config[option] = value; + }); + + return config; +} + +// *************************************************************** +// Public functions start here. +// These are available via the 'state' object. Their context ('this' +// keyword) is the 'state' object. The magic private function that makes +// them available and sets up their context is makeFunctionsPublic(). +// *************************************************************** + +// function bindTo(methodsDict, obj, context, rewrite) +// Creates a new function with specific context and assigns it to the provided +// object. +// eslint-disable-next-line no-shadow +function bindTo(methodsDict, obj, context, rewrite) { + $.each(methodsDict, function(name, method) { + if (_.isFunction(method)) { + if (_.isUndefined(rewrite)) { + rewrite = true; + } + + if (_.isUndefined(obj[name]) || rewrite) { + obj[name] = _.bind(method, context); + } + } + }); +} + +function loadYoutubePlayer() { + if (this.htmlPlayerLoaded) { return; } + + console.log( + '[Video info]: Fetch metadata for YouTube video.' + ); + + this.fetchMetadata(); + this.parseSpeed(); +} + +function loadHtmlPlayer() { + // When the youtube link doesn't work for any reason + // (for example, firewall) any + // alternate sources should automatically play. + if (!_prepareHTML5Video(this)) { + console.log( + '[Video info]: Continue loading ' + + 'YouTube video.' + ); + + // Non-YouTube sources were not found either. + + this.el.find('.video-player div') + .removeClass('hidden'); + this.el.find('.video-player .video-error') + .addClass('is-hidden'); + + // If in reality the timeout was to short, try to + // continue loading the YouTube video anyways. + this.loadYoutubePlayer(); + } else { + console.log( + '[Video info]: Start HTML5 player.' + ); + + // In-browser HTML5 player does not support quality + // control. + this.el.find('.quality_control').hide(); + _renderElements(this); + } +} + +// function initialize(element) +// The function set initial configuration and preparation. + +function initialize(element) { + let self = this, + el = this.el, + id = this.id, + container = el.find('.video-wrapper'), + __dfd__ = $.Deferred(), + isTouch = onTouchBasedDevice() || ''; + + if (isTouch) { + el.addClass('is-touch'); + } + + $.extend(this, { + __dfd__: __dfd__, + container: container, + isFullScreen: false, + isTouch: isTouch + }); + + console.log('[Video info]: Initializing video with id "%s".', id); + + // We store all settings passed to us by the server in one place. These + // are "read only", so don't modify them. All variable content lives in + // 'state' object. + // jQuery .data() return object with keys in lower camelCase format. + this.config = $.extend({}, _getConfiguration(this.metadata, this.storage), { + element: element, + fadeOutTimeout: 1400, + captionsFreezeTime: 10000, + mode: $.cookie('edX_video_player_mode'), + // Available HD qualities will only be accessible once the video has + // been played once, via player.getAvailableQualityLevels. + availableHDQualities: [] + }); + + if (this.config.endTime < this.config.startTime) { + this.config.endTime = null; + } + + this.lang = this.config.transcriptLanguage; + this.speed = this.speedToString( + this.config.speed || this.config.generalSpeed + ); + this.auto_advance = this.config.autoAdvance; + this.htmlPlayerLoaded = false; + this.duration = this.metadata.duration; + + _setConfigurations(this); + + // If `prioritizeHls` is set to true than `hls` is the primary playback + if (this.config.prioritizeHls || !(_parseYouTubeIDs(this))) { + // If we do not have YouTube ID's, try parsing HTML5 video sources. + if (!_prepareHTML5Video(this)) { + __dfd__.reject(); + // Non-YouTube sources were not found either. + return __dfd__.promise(); + } + + console.log('[Video info]: Start player in HTML5 mode.'); + _renderElements(this); + } else { + _renderElements(this); + + _waitForYoutubeApi(this); + + let scriptTag = document.createElement('script'); + + scriptTag.src = this.config.ytApiUrl; + scriptTag.async = true; + + $(scriptTag).on('load', function() { + self.loadYoutubePlayer(); + }); + $(scriptTag).on('error', function() { + console.log( + '[Video info]: YouTube returned an error for ' + + 'video with id "' + self.id + '".' + ); + // If the video is already loaded in `_waitForYoutubeApi` by the + // time we get here, then we shouldn't load it again. + if (!self.htmlPlayerLoaded) { + self.loadHtmlPlayer(); + } + }); + + window.Video.loadYouTubeIFrameAPI(scriptTag); + } + return __dfd__.promise(); +} + +// function parseYoutubeStreams(state, youtubeStreams) +// +// Take a string in the form: +// "iCawTYPtehk:0.75,KgpclqP-LBA:1.0,9-2670d5nvU:1.5" +// parse it, and make it available via the 'state' object. If we are +// not given a string, or it's length is zero, then we return false. +// +// @return +// false: We don't have YouTube video IDs to work with; most likely +// we have HTML5 video sources. +// true: Parsing of YouTube video IDs went OK, and we can proceed +// onwards to play YouTube videos. +function parseYoutubeStreams(youtubeStreams) { + if (_.isUndefined(youtubeStreams) || !youtubeStreams.length) { + return false; + } + + this.videos = {}; + + _.each(youtubeStreams.split(/,/), function(video) { + let speed; + video = video.split(/:/); + speed = this.speedToString(video[0]); + this.videos[speed] = video[1]; + }, this); + + return _.isString(this.videos['1.0']); +} + +// function fetchMetadata() +// +// When dealing with YouTube videos, we must fetch meta data that has +// certain key facts not available while the video is loading. For +// example the length of the video can be determined from the meta +// data. +function fetchMetadata() { + let self = this, + metadataXHRs = []; + + this.metadata = {}; + + metadataXHRs = _.map(this.videos, function(url, speed) { + return self.getVideoMetadata(url, function(data) { + if (data.items.length > 0) { + let metaDataItem = data.items[0]; + self.metadata[metaDataItem.id] = metaDataItem.contentDetails; + } + }); + }); + + $.when.apply(this, metadataXHRs).done(function() { + self.el.trigger('metadata_received'); + + // Not only do we trigger the "metadata_received" event, we also + // set a flag to notify that metadata has been received. This + // allows for code that will miss the "metadata_received" event + // to know that metadata has been received. This is important in + // cases when some code will subscribe to the "metadata_received" + // event after it has been triggered. + self.youtubeMetadataReceived = true; + }); +} + +// function parseSpeed() +// +// Create a separate array of available speeds. +function parseSpeed() { + this.speeds = _.keys(this.videos).sort(); +} + +function setSpeed(newSpeed) { + // Possible speeds for each player type. + // HTML5 = [0.75, 1, 1.25, 1.5, 2] + // Youtube Flash = [0.75, 1, 1.25, 1.5] + // Youtube HTML5 = [0.25, 0.5, 1, 1.5, 2] + let map = { + 0.25: '0.75', // Youtube HTML5 -> HTML5 or Youtube Flash + '0.50': '0.75', // Youtube HTML5 -> HTML5 or Youtube Flash + 0.75: '0.50', // HTML5 or Youtube Flash -> Youtube HTML5 + 1.25: '1.50', // HTML5 or Youtube Flash -> Youtube HTML5 + 2.0: '1.50' // HTML5 or Youtube HTML5 -> Youtube Flash + }; + + if (_.contains(this.speeds, newSpeed)) { + this.speed = newSpeed; + } else { + newSpeed = map[newSpeed]; + this.speed = _.contains(this.speeds, newSpeed) ? newSpeed : '1.0'; + } + this.speed = parseFloat(this.speed); +} + +function setAutoAdvance(enabled) { + this.auto_advance = enabled; +} + +function getVideoMetadata(url, callback) { + let youTubeEndpoint; + if (!(_.isString(url))) { + url = this.videos['1.0'] || ''; + } + // Will hit the API URL to get the youtube video metadata. + youTubeEndpoint = this.config.ytMetadataEndpoint; // The new runtime supports anonymous users + // and uses an XBlock handler to get YouTube metadata + if (!youTubeEndpoint) { + // The old runtime has a full/separate LMS API for getting YouTube metadata, but it doesn't + // support anonymous users nor videos that play in a sandboxed iframe. + youTubeEndpoint = [this.config.lmsRootURL, '/courses/yt_video_metadata', '?id=', url].join(''); + } + return $.ajax({ + url: youTubeEndpoint, + success: _.isFunction(callback) ? callback : null, + error: function() { + console.warn( + 'Unable to get youtube video metadata. Some video metadata may be unavailable.' + ); + }, + notifyOnError: false + }); +} + +function youtubeId(speed) { + let currentSpeed = this.isFlashMode() ? this.speed : '1.0'; + + return this.videos[speed] + || this.videos[currentSpeed] + || this.videos['1.0']; +} + +function getDuration() { + try { + let safeMoment = typeof moment !== 'undefined' ? moment : window.moment; + return safeMoment.duration(this.metadata[this.youtubeId()].duration, safeMoment.ISO_8601).asSeconds(); + } catch (err) { + return _.result(this.metadata[this.youtubeId('1.0')], 'duration') || 0; + } +} + +/** + * Sets player mode. + * + * @param {string} mode Mode to set for the video player if it is supported. + * Otherwise, `html5` is used by default. + */ +function setPlayerMode(mode) { + let supportedModes = ['html5', 'flash']; + + mode = _.contains(supportedModes, mode) ? mode : 'html5'; + this.currentPlayerMode = mode; +} + +/** + * Returns current player mode. + * + * @return {string} Returns string that describes player mode + */ +function getPlayerMode() { + return this.currentPlayerMode; +} + +/** + * Checks if current player mode is Flash. + * + * @return {boolean} Returns `true` if current mode is `flash`, otherwise + * it returns `false` + */ +function isFlashMode() { + return this.getPlayerMode() === 'flash'; +} + +/** + * Checks if current player mode is Html5. + * + * @return {boolean} Returns `true` if current mode is `html5`, otherwise + * it returns `false` + */ +function isHtml5Mode() { + return this.getPlayerMode() === 'html5'; +} + +function isYoutubeType() { + return this.videoType === 'youtube'; +} + +function speedToString(speed) { + return parseFloat(speed).toFixed(2).replace(/\.00$/, '.0'); +} + +function getCurrentLanguage() { + let keys = _.keys(this.config.transcriptLanguages); + + if (keys.length) { + if (!_.contains(keys, this.lang)) { + if (_.contains(keys, 'en')) { + this.lang = 'en'; + } else { + this.lang = keys.pop(); + } + } + } else { + return null; + } + + return this.lang; +} + +/* + * The trigger() function will assume that the @objChain is a complete + * chain with a method (function) at the end. It will call this function. + * So for example, when trigger() is called like so: + * + * state.trigger('videoPlayer.pause', {'param1': 10}); + * + * Then trigger() will execute: + * + * state.videoPlayer.pause({'param1': 10}); + */ +function trigger(objChain) { + let extraParameters = Array.prototype.slice.call(arguments, 1), + i, tmpObj, chain; + + // Remember that 'this' is the 'state' object. + tmpObj = this; + chain = objChain.split('.'); + + // At the end of the loop the variable 'tmpObj' will either be the + // correct object/function to trigger/invoke. If the 'chain' chain of + // object is incorrect (one of the link is non-existent), then the loop + // will immediately exit. + while (chain.length) { + i = chain.shift(); + + if (tmpObj.hasOwnProperty(i)) { + tmpObj = tmpObj[i]; + } else { + // An incorrect object chain was specified. + + return false; + } + } + + tmpObj.apply(this, extraParameters); + + return true; +} diff --git a/xmodule/assets/video/public/js/025_focus_grabber.js b/xmodule/assets/video/public/js/025_focus_grabber.js new file mode 100644 index 0000000000..48ec5527ad --- /dev/null +++ b/xmodule/assets/video/public/js/025_focus_grabber.js @@ -0,0 +1,132 @@ +/* + * 025_focus_grabber.js + * + * Purpose: Provide a way to focus on autohidden Video controls. + * + * + * Because in HTML player mode we have a feature of autohiding controls on + * mouse inactivity, sometimes focus is lost from the currently selected + * control. What's more, when all controls are autohidden, we can't get to any + * of them because by default browser does not place hidden elements on the + * focus chain. + * + * To get around this minor annoyance, this module will manage 2 placeholder + * elements that will be invisible to the user's eye, but visible to the + * browser. This will allow for a sneaky stealing of focus and placing it where + * we need (on hidden controls). + * + * This code has been moved to a separate module because it provides a concrete + * block of functionality that can be turned on (off). + */ + +/* + * "If you want to climb a mountain, begin at the top." + * + * ~ Zen saying + */ + + + +// FocusGrabber module. +let FocusGrabber = function(state) { + let dfd = $.Deferred(); + + state.focusGrabber = {}; + + _makeFunctionsPublic(state); + _renderElements(state); + _bindHandlers(state); + + dfd.resolve(); + return dfd.promise(); +}; + +// Private functions. + +function _makeFunctionsPublic(state) { + let methodsDict = { + disableFocusGrabber: disableFocusGrabber, + enableFocusGrabber: enableFocusGrabber, + onFocus: onFocus + }; + + state.bindTo(methodsDict, state.focusGrabber, state); +} + +function _renderElements(state) { + state.focusGrabber.elFirst = state.el.find('.focus_grabber.first'); + state.focusGrabber.elLast = state.el.find('.focus_grabber.last'); + + // From the start, the Focus Grabber must be disabled so that + // tabbing (switching focus) does not land the user on one of the + // placeholder elements (elFirst, elLast). + state.focusGrabber.disableFocusGrabber(); +} + +function _bindHandlers(state) { + state.focusGrabber.elFirst.on('focus', state.focusGrabber.onFocus); + state.focusGrabber.elLast.on('focus', state.focusGrabber.onFocus); + + // When the video container element receives programmatic focus, then + // on un-focus ('blur' event) we should trigger a 'mousemove' event so + // as to reveal autohidden controls. + state.el.on('blur', function() { + state.el.trigger('mousemove'); + }); +} + +// Public functions. + +function enableFocusGrabber() { + let tabIndex; + + // When the Focus Grabber is being enabled, there are two different + // scenarios: + // + // 1.) Currently focused element was inside the video player. + // 2.) Currently focused element was somewhere else on the page. + // + // In the first case we must make sure that the video player doesn't + // loose focus, even though the controls are autohidden. + if ($(document.activeElement).parents().hasClass('video')) { + tabIndex = -1; + } else { + tabIndex = 0; + } + + this.focusGrabber.elFirst.attr('tabindex', tabIndex); + this.focusGrabber.elLast.attr('tabindex', tabIndex); + + // Don't loose focus. We are inside video player on some control, but + // because we can't remain focused on a hidden element, we will shift + // focus to the main video element. + // + // Once the main element will receive the un-focus ('blur') event, a + // 'mousemove' event will be triggered, and the video controls will + // receive focus once again. + if (tabIndex === -1) { + this.el.focus(); + + this.focusGrabber.elFirst.attr('tabindex', 0); + this.focusGrabber.elLast.attr('tabindex', 0); + } +} + +function disableFocusGrabber() { + // Only programmatic focusing on these elements will be available. + // We don't want the user to focus on them (for example with the 'Tab' + // key). + this.focusGrabber.elFirst.attr('tabindex', -1); + this.focusGrabber.elLast.attr('tabindex', -1); +} + +function onFocus(event, params) { + // Once the Focus Grabber placeholder elements will gain focus, we will + // trigger 'mousemove' event so that the autohidden controls will + // become visible. + this.el.trigger('mousemove'); + + this.focusGrabber.disableFocusGrabber(); +} + +export default FocusGrabber; diff --git a/xmodule/assets/video/public/js/02_html5_hls_video.js b/xmodule/assets/video/public/js/02_html5_hls_video.js new file mode 100644 index 0000000000..ddc198bc72 --- /dev/null +++ b/xmodule/assets/video/public/js/02_html5_hls_video.js @@ -0,0 +1,145 @@ +/* eslint-disable no-console, no-param-reassign */ +/** + * HTML5 video player module to support HLS video playback. + * + */ + +'use strict'; + +import _ from 'underscore'; +import HTML5Video from './02_html5_video.js'; +import HLS from 'hls'; + +let HLSVideo = {}; + +HLSVideo.Player = (function() { + /** + * Initialize HLS video player. + * + * @param {jQuery} el Reference to video player container element + * @param {Object} config Contains common config for video player + */ + function Player(el, config) { + let self = this; + + this.config = config; + + // do common initialization independent of player type + this.init(el, config); + + _.bindAll(this, 'playVideo', 'pauseVideo', 'onReady'); + + // If we have only HLS sources and browser doesn't support HLS then show error message. + if (config.HLSOnlySources && !config.canPlayHLS) { + this.showErrorMessage(null, '.video-hls-error'); + return; + } + + this.config.state.el.on('initialize', _.once(function() { + console.log('[HLS Video]: HLS Player initialized'); + self.showPlayButton(); + })); + + // Safari has native support to play HLS videos + if (config.browserIsSafari) { + this.videoEl.attr('src', config.videoSources[0]); + } else { + // load auto start if auto_advance is enabled + if (config.state.auto_advance) { + this.hls = new HLS({autoStartLoad: true}); + } else { + this.hls = new HLS({autoStartLoad: false}); + } + this.hls.loadSource(config.videoSources[0]); + this.hls.attachMedia(this.video); + + this.hls.on(HLS.Events.ERROR, this.onError.bind(this)); + + this.hls.on(HLS.Events.MANIFEST_PARSED, function(event, data) { + console.log( + '[HLS Video]: MANIFEST_PARSED, qualityLevelsInfo: ', + data.levels.map(function(level) { + return { + bitrate: level.bitrate, + resolution: level.width + 'x' + level.height + }; + }) + ); + self.config.onReadyHLS(); + }); + this.hls.on(HLS.Events.LEVEL_SWITCHED, function(event, data) { + let level = self.hls.levels[data.level]; + console.log( + '[HLS Video]: LEVEL_SWITCHED, qualityLevelInfo: ', + { + bitrate: level.bitrate, + resolution: level.width + 'x' + level.height + } + ); + }); + } + } + + Player.prototype = Object.create(HTML5Video.Player.prototype); + Player.prototype.constructor = Player; + + Player.prototype.playVideo = function() { + HTML5Video.Player.prototype.updatePlayerLoadingState.apply(this, ['show']); + if (!this.config.browserIsSafari) { + this.hls.startLoad(); + } + HTML5Video.Player.prototype.playVideo.apply(this); + }; + + Player.prototype.pauseVideo = function() { + HTML5Video.Player.prototype.pauseVideo.apply(this); + HTML5Video.Player.prototype.updatePlayerLoadingState.apply(this, ['hide']); + }; + + Player.prototype.onPlaying = function() { + HTML5Video.Player.prototype.onPlaying.apply(this); + HTML5Video.Player.prototype.updatePlayerLoadingState.apply(this, ['hide']); + }; + + Player.prototype.onReady = function() { + this.config.events.onReady(null); + }; + + /** + * Handler for HLS video errors. This only takes care of fatal erros, non-fatal errors + * are automatically handled by hls.js + * + * @param {String} event `hlsError` + * @param {Object} data Contains the information regarding error occurred. + */ + Player.prototype.onError = function(event, data) { + if (data.fatal) { + switch (data.type) { + case HLS.ErrorTypes.NETWORK_ERROR: + console.error( + '[HLS Video]: Fatal network error encountered, try to recover. Details: %s', + data.details + ); + this.hls.startLoad(); + break; + case HLS.ErrorTypes.MEDIA_ERROR: + console.error( + '[HLS Video]: Fatal media error encountered, try to recover. Details: %s', + data.details + ); + this.hls.recoverMediaError(); + break; + default: + console.error( + '[HLS Video]: Unrecoverable error encountered. Details: %s', + data.details + ); + break; + } + } + }; + + return Player; +}()); + +export default HLSVideo; diff --git a/xmodule/assets/video/public/js/02_html5_video.js b/xmodule/assets/video/public/js/02_html5_video.js new file mode 100644 index 0000000000..8393720543 --- /dev/null +++ b/xmodule/assets/video/public/js/02_html5_video.js @@ -0,0 +1,380 @@ +/* eslint-disable no-console, no-param-reassign */ +/** + * @file HTML5 video player module. Provides methods to control the in-browser + * HTML5 video player. + * + * The goal was to write this module so that it closely resembles the YouTube + * API. The main reason for this is because initially the edX video player + * supported only YouTube videos. When HTML5 support was added, for greater + * compatibility, and to reduce the amount of code that needed to be modified, + * it was decided to write a similar API as the one provided by YouTube. + * + * @module HTML5Video + */ + +import _ from 'underscore'; + +let HTML5Video = {}; + +HTML5Video.Player = (function() { + /* + * Constructor function for HTML5 Video player. + * + * @param {String|Object} el A DOM element where the HTML5 player will + * be inserted (as returned by jQuery(selector) function), or a + * selector string which will be used to select an element. This is a + * required parameter. + * + * @param config - An object whose properties will be used as + * configuration options for the HTML5 video player. This is an + * optional parameter. In the case if this parameter is missing, or + * some of the config object's properties are missing, defaults will be + * used. The available options (and their defaults) are as + * follows: + * + * config = { + * + * videoSources: [], // An array with properties being video + * // sources. The property name is the + * // video format of the source. Supported + * // video formats are: 'mp4', 'webm', and + * // 'ogg'. + * poster: Video poster URL + * + * browserIsSafari: Flag to tell if current browser is Safari + * + * events: { // Object's properties identify the + * // events that the API fires, and the + * // functions (event listeners) that the + * // API will call when those events occur. + * // If value is null, or property is not + * // specified, then no callback will be + * // called for that event. + * + * onReady: null, + * onStateChange: null + * } + * } + */ + function Player(el, config) { + let errorMessage, lastSource, sourceList; + + // Create HTML markup for individual sources of the HTML5