Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin' into katrinan029/upgrade-edx-enterprise-7316111

Browse Source
This commit is contained in:
Katrina Nguyen
2024-10-09 15:56:51 +00:00
parent 0a4d24adec 243b1b4e2e
commit d7f7ca17a5
6 changed files with 274 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
.github/CODEOWNERS vendored
View File

@@ -15,6 +15,7 @@ lms/djangoapps/grades/
lms/djangoapps/instructor/
lms/djangoapps/instructor_task/
lms/djangoapps/mobile_api/
openedx/core/djangoapps/commerce/ @openedx/2u-infinity
openedx/core/djangoapps/credentials @openedx/2U-aperture
openedx/core/djangoapps/credit @openedx/2U-aperture
openedx/core/djangoapps/enrollments/ @openedx/2U-aperture
@@ -22,6 +23,7 @@ openedx/core/djangoapps/heartbeat/
openedx/core/djangoapps/oauth_dispatch
openedx/core/djangoapps/user_api/ @openedx/2U-aperture
openedx/core/djangoapps/user_authn/ @openedx/2U-vanguards
openedx/core/djangoapps/verified_track_content/ @openedx/2u-infinity
openedx/features/course_experience/
xmodule/
@@ -36,16 +38,18 @@ common/djangoapps/track/
lms/djangoapps/certificates/ @openedx/2U-aperture
# Discovery
common/djangoapps/course_modes/
common/djangoapps/course_modes/ @openedx/2U-aperture
common/djangoapps/enrollment/
lms/djangoapps/branding/ @openedx/2U-aperture
lms/djangoapps/commerce/
lms/djangoapps/experiments/ @openedx/2U-aperture
lms/djangoapps/learner_dashboard/ @openedx/2U-aperture
lms/djangoapps/learner_home/ @openedx/2U-aperture
openedx/features/content_type_gating/
common/djangoapps/entitlements/ @openedx/2U-aperture
lms/djangoapps/branding/ @openedx/2U-aperture
lms/djangoapps/commerce/ @openedx/2u-infinity
lms/djangoapps/experiments/ @openedx/2u-infinity
lms/djangoapps/gating/ @openedx/2u-infinity
lms/djangoapps/learner_dashboard/ @openedx/2U-aperture
lms/djangoapps/learner_home/ @openedx/2U-aperture
openedx/features/content_type_gating/ @openedx/2u-infinity
openedx/features/course_duration_limits/
openedx/features/discounts/
openedx/features/discounts/ @openedx/2u-infinity
# Ping Axim On-call if someone uses the QuickStart
# https://docs.openedx.org/en/latest/developers/quickstarts/first_openedx_pr.html

13
lms/djangoapps/discussion/rest_api/discussions_notifications.py
View File

@@ -392,7 +392,8 @@ def clean_thread_html_body(html_body):
"video", "track", # Video Tags
"audio", # Audio Tags
"embed", "object", "iframe", # Embedded Content
"script"
"script",
"b", "strong", "i", "em", "u", "s", "strike", "del", "ins", "mark", "sub", "sup", # Text Formatting
]
# Remove the specified tags while keeping their content
@@ -403,9 +404,10 @@ def clean_thread_html_body(html_body):
# Replace tags that are not allowed in email
tags_to_update = [
{"source": "button", "target": "span"},
{"source": "h1", "target": "h4"},
{"source": "h2", "target": "h4"},
{"source": "h3", "target": "h4"},
*[
{"source": tag, "target": "p"}
for tag in ["div", "section", "article", "h1", "h2", "h3", "h4", "h5", "h6"]
],
]
for tag_dict in tags_to_update:
for source_tag in html_body.find_all(tag_dict['source']):
@@ -414,4 +416,7 @@ def clean_thread_html_body(html_body):
target_tag.string = source_tag.string
source_tag.replace_with(target_tag)
for tag in html_body.find_all(True):
tag.attrs = {}
tag['style'] = 'margin: 0'
return str(html_body)

50
lms/djangoapps/discussion/rest_api/tests/test_discussions_notifications.py
View File

@@ -104,14 +104,14 @@ class TestCleanThreadHtmlBody(unittest.TestCase):
<p>This is a <a href="#">link</a> to a page.</p>
<p>Here is an image: <img src="image.jpg" alt="image"></p>
<p>Embedded video: <iframe src="video.mp4"></iframe></p>
<p>Script test: <script>alert('hello');</script></p>
<p>Script test: <script>alert("hello");</script></p>
<p>Some other content that should remain.</p>
"""
expected_output = ("<p>This is a link to a page.</p>"
"<p>Here is an image: </p>"
"<p>Embedded video: </p>"
"<p>Script test: alert('hello');</p>"
"<p>Some other content that should remain.</p>")
expected_output = ('<p style="margin: 0">This is a link to a page.</p>'
'<p style="margin: 0">Here is an image: </p>'
'<p style="margin: 0">Embedded video: </p>'
'<p style="margin: 0">Script test: alert("hello");</p>'
'<p style="margin: 0">Some other content that should remain.</p>')
result = clean_thread_html_body(html_body)
@@ -132,19 +132,16 @@ class TestCleanThreadHtmlBody(unittest.TestCase):
"""
Test that the clean_thread_html_body function truncates the HTML body to 500 characters
"""
html_body = """
<p>This is a long text that should be truncated to 500 characters.</p>
""" * 20 # Repeat to exceed 500 characters
result = clean_thread_html_body(html_body)
self.assertGreaterEqual(500, len(result))
html_body = "This is a long text that should be truncated to 500 characters." * 20
result = clean_thread_html_body(f"<p>{html_body}</p>")
self.assertGreaterEqual(525, len(result)) # 500 characters + 25 characters for the HTML tags
def test_no_tags_to_remove(self):
"""
Test that the clean_thread_html_body function does not remove any tags if there are no unwanted tags
"""
html_body = "<p>This paragraph has no tags to remove.</p>"
expected_output = "<p>This paragraph has no tags to remove.</p>"
expected_output = '<p style="margin: 0">This paragraph has no tags to remove.</p>'
result = clean_thread_html_body(html_body)
self.assertEqual(result, expected_output)
@@ -169,28 +166,33 @@ class TestCleanThreadHtmlBody(unittest.TestCase):
result = clean_thread_html_body(html_body)
self.assertEqual(result.strip(), expected_output)
def test_tag_replace(self):
"""
Tests if the clean_thread_html_body function replaces tags
"""
for tag in ["div", "section", "article", "h1", "h2", "h3", "h4", "h5", "h6"]:
html_body = f'<{tag}>Text</{tag}>'
result = clean_thread_html_body(html_body)
self.assertEqual(result, '<p style="margin: 0">Text</p>')
def test_button_tag_replace(self):
"""
Tests that the clean_thread_html_body function replaces the button tag with span tag
"""
# Tests for button replacement tag with text
html_body = '<button class="abc">Button</button>'
expected_output = '<span class="abc">Button</span>'
expected_output = '<span style="margin: 0">Button</span>'
result = clean_thread_html_body(html_body)
self.assertEqual(result, expected_output)
# Tests button tag replacement without text
html_body = '<button class="abc"></button>'
expected_output = '<span class="abc"></span>'
expected_output = '<span style="margin: 0"></span>'
result = clean_thread_html_body(html_body)
self.assertEqual(result, expected_output)
def test_heading_tag_replace(self):
"""
Tests that the clean_thread_html_body function replaces the h1, h2 and h3 tags with h4 tag
"""
for tag in ['h1', 'h2', 'h3']:
html_body = f'<{tag}>Heading</{tag}>'
expected_output = '<h4>Heading</h4>'
result = clean_thread_html_body(html_body)
self.assertEqual(result, expected_output)
def test_attributes_removal_from_tag(self):
# Tests for removal of attributes from tags
html_body = '<p class="abc" style="color:red" aria-disabled=true>Paragraph</p>'
result = clean_thread_html_body(html_body)
self.assertEqual(result, '<p style="margin: 0">Paragraph</p>')

34
lms/djangoapps/verify_student/management/commands/approve_id_verifications.py
View File

@@ -10,9 +10,11 @@ from pprint import pformat
from django.core.management.base import BaseCommand, CommandError
from common.djangoapps.student.models_api import get_name, get_pending_name_change
from lms.djangoapps.verify_student.api import send_approval_email
from lms.djangoapps.verify_student.models import SoftwareSecurePhotoVerification
from lms.djangoapps.verify_student.utils import earliest_allowed_verification_date
from openedx.features.name_affirmation_api.utils import get_name_affirmation_service
log = logging.getLogger(__name__)
@@ -149,5 +151,37 @@ class Command(BaseCommand):
for verification in existing_id_verifications:
verification.approve(service='idv_verifications command')
send_approval_email(verification)
self._approve_verified_name_for_software_secure_verification(verification)
return list(failed_user_ids)
def _approve_verified_name_for_software_secure_verification(self, verification):
"""
This method manually creates a verified name given a SoftwareSecurePhotoVerification object.
"""
name_affirmation_service = get_name_affirmation_service()
if name_affirmation_service:
from edx_name_affirmation.exceptions import VerifiedNameDoesNotExist # pylint: disable=import-error
pending_name_change = get_pending_name_change(verification.user)
if pending_name_change:
full_name = pending_name_change.new_name
else:
full_name = get_name(verification.user.id)
try:
name_affirmation_service.update_verified_name_status(
verification.user,
'approved',
verification_attempt_id=verification.id
)
except VerifiedNameDoesNotExist:
name_affirmation_service.create_verified_name(
verification.user,
verification.name,
full_name,
verification_attempt_id=verification.id,
status='approved',
)

59
lms/djangoapps/verify_student/management/commands/tests/test_approve_id_verifications.py
View File

@@ -6,6 +6,8 @@ import ddt
import logging
import os
import tempfile
from unittest import skipUnless
from unittest.mock import MagicMock, patch
import pytest
from django.core import mail
@@ -15,9 +17,12 @@ from testfixtures import LogCapture
from common.djangoapps.student.tests.factories import UserFactory, UserProfileFactory
from lms.djangoapps.verify_student.models import SoftwareSecurePhotoVerification
from openedx.features.name_affirmation_api.utils import get_name_affirmation_service
LOGGER_NAME = 'lms.djangoapps.verify_student.management.commands.approve_id_verifications'
name_affirmation_service = get_name_affirmation_service()
@ddt.ddt
class TestApproveIDVerificationsCommand(TestCase):
@@ -158,3 +163,57 @@ class TestApproveIDVerificationsCommand(TestCase):
"""
with pytest.raises(CommandError):
call_command('approve_id_verifications', 'invalid/user_id/file/path')
@skipUnless(name_affirmation_service is not None, 'Requires Name Affirmation')
@patch('lms.djangoapps.verify_student.management.commands.approve_id_verifications.get_name_affirmation_service')
def test_create_verified_names(self, mock_get_service):
mock_service = MagicMock()
mock_get_service.return_value = mock_service
verification = SoftwareSecurePhotoVerification.objects.create(
user=self.user1_profile.user,
name=self.user1_profile.name,
status='submitted',
)
call_command('approve_id_verifications', self.tmp_file_path)
mock_service.update_verified_name_status.assert_called_with(
self.user1_profile.user,
'approved',
verification_attempt_id=verification.id,
)
@skipUnless(name_affirmation_service is not None, 'Requires Name Affirmation')
@patch('lms.djangoapps.verify_student.management.commands.approve_id_verifications.get_name')
@patch('lms.djangoapps.verify_student.management.commands.approve_id_verifications.get_pending_name_change')
@patch('lms.djangoapps.verify_student.management.commands.approve_id_verifications.get_name_affirmation_service')
@ddt.data(
'',
MagicMock(new_name='test')
)
def test_create_update_verified_names(self, pending_name, mock_get_service, mock_get_pending, mock_get_name):
from edx_name_affirmation.exceptions import VerifiedNameDoesNotExist # pylint: disable=import-error
mock_service = MagicMock()
mock_get_service.return_value = mock_service
mock_service.update_verified_name_status.side_effect = VerifiedNameDoesNotExist()
mock_get_pending.return_value = pending_name
mock_get_name.return_value = self.user1_profile.name
verification = SoftwareSecurePhotoVerification.objects.create(
user=self.user1_profile.user,
name=self.user1_profile.name,
status='submitted',
)
expected_name = 'test' if pending_name else self.user1_profile.name
call_command('approve_id_verifications', self.tmp_file_path)
mock_service.create_verified_name.assert_called_with(
verification.user,
verification.name,
expected_name,
verification_attempt_id=verification.id,
status='approved',
)

218
requirements/constraints.txt
View File

@@ -7,7 +7,8 @@
# link to other information that will help people in the future to remove the
# pin when possible. Writing an issue against the offending project and
# linking to it here is good.
# For further details on how to properly write constraints here please consult
# https://openedx.atlassian.net/wiki/spaces/COMM/pages/4400250883/Adding+pinned+dependencies+in+constraint+file
# This file contains all common constraints for edx-repos
-c common_constraints.txt
@@ -18,127 +19,176 @@
# Ticket: https://github.com/openedx/edx-platform/issues/35334
algoliasearch<4.0.0
# Date: 2024-03-14
# Temporary to Support the python 3.11 Upgrade
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35281
backports.zoneinfo;python_version<"3.9" # Newer versions have zoneinfo available in the standard library
# Date: 2020-02-26
# As it is not clarified what exact breaking changes will be introduced as per
# the next major release, ensure the installed version is within boundaries.
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35280
celery>=5.2.2,<6.0.0
# Date: 2021-05-17
# greater version breaking upgrade builds
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35279
click==8.1.6
# Date: 2022-07-20
# edx-enterprise, snowflake-connector-python require charset-normalizer==2.0.0
# Can be removed once snowflake-connector-python>2.7.9 is released with the fix.
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35278
charset-normalizer<2.1.0
# Date: 2024-02-02
# Stay on LTS version, remove once this is added to common constraint
Django<5.0
# Date: 2020-02-10
# django-oauth-toolkit version >=2.0.0 has breaking changes. More details
# mentioned on this issue https://github.com/openedx/edx-platform/issues/32884
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35277
django-oauth-toolkit==1.7.1
# Date: 2024-02-02
# incremental upgrade
django-simple-history==3.4.0
# Date: 2021-05-17
# greater version has breaking changes and requires some migration steps.
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35276
django-webpack-loader==0.7.0
# Date: 2023-06-20
# Adding pin to avoid any major upgrade
djangorestframework<3.15.0
# Date: 2023-07-19
# The version of django-stubs we can use depends on which Django release we're using
# 1.16.0 works with Django 3.2 through 4.1
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35275
django-stubs==1.16.0
djangorestframework-stubs==3.14.0 # Pinned to match django-stubs. Remove this when we can remove the above pin.
# Date: 2024-07-23
# django-storages==1.14.4 breaks course imports
# Two lines were added in 1.14.4 that make file_exists_in_storage function always return False,
# as the default value of AWS_S3_FILE_OVERWRITE is True
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35170
django-storages<1.14.4
# Date: 2019-08-16
# The team that owns this package will manually bump this package rather than having it pulled in automatically.
# This is to allow them to better control its deployment and to do it in a process that works better
# for them.
edx-enterprise==4.27.2
# Stay on LTS version, remove once this is added to common constraint
Django<5.0
# django-oauth-toolkit version >=2.0.0 has breaking changes. More details
# mentioned on this issue https://github.com/openedx/edx-platform/issues/32884
django-oauth-toolkit==1.7.1
# incremental upgrade
django-simple-history==3.4.0
# Adding pin to avoid any major upgrade
pymongo<4.4.1
# Date: 2024-05-09
# This has to be constrained as well because newer versions of edx-i18n-tools need the
# newer version of lxml but that requirement was not made expilict in the 1.6.0 version
# of the package. This can be un-pinned when we're upgrading lxml.
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35274
edx-i18n-tools<1.6.0
# Date: 2024-07-26
# To override the constraint of edx-lint
# This can be removed once https://github.com/openedx/edx-platform/issues/34586 is resolved
# and the upstream constraint in edx-lint has been removed.
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35273
event-tracking==3.0.0
# greater version has breaking changes and requires some migration steps.
django-webpack-loader==0.7.0
# At the time of writing this comment, we do not know whether py2neo>=2022
# will support our currently-deployed Neo4j version (3.5).
# Feel free to loosen this constraint if/when it is confirmed that a later
# version of py2neo will work with Neo4j 3.5.
py2neo<2022
# edx-enterprise, snowflake-connector-python require charset-normalizer==2.0.0
# Can be removed once snowflake-connector-python>2.7.9 is released with the fix.
charset-normalizer<2.1.0
# markdown>=3.4.0 has failures due to internal refactorings which causes the tests to fail
# pinning the version untill the issue gets resolved in the package itself
markdown<3.4.0
# pycodestyle==2.9.0 generates false positive error E275.
# Constraint can be removed once the issue https://github.com/PyCQA/pycodestyle/issues/1090 is fixed.
pycodestyle<2.9.0
pylint<2.16.0 # greater version failing quality test. Fix them in seperate ticket.
# urllib3>=2.0.0 conflicts with elastic search && snowflake-connector-python packages
# which require urllib3<2 for now.
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/32222
urllib3<2.0.0
# Adding pin to avoid any major upgrade
djangorestframework<3.15.0
# The version of django-stubs we can use depends on which Django release we're using
# 1.16.0 works with Django 3.2 through 4.1
django-stubs==1.16.0
djangorestframework-stubs==3.14.0 # Pinned to match django-stubs. Remove this when we can remove the above pin.
# Date: 2023-07-26
# Our legacy Sass code is incompatible with anything except this ancient libsass version.
# Here is a ticket to upgrade, but it's of debatable importance given that we are rapidly moving
# away from legacy LMS/CMS frontends:
# https://github.com/openedx/edx-platform/issues/31616
libsass==0.10.0
# greater version breaking upgrade builds
click==8.1.6
# pinning this version to avoid updates while the library is being developed
openedx-learning==0.13.1
# Open AI version 1.0.0 dropped support for openai.ChatCompletion which is currently in use in enterprise.
openai<=0.28.1
# optimizely-sdk 5.0.0 is breaking following test with segmentation fault
# common/djangoapps/third_party_auth/tests/test_views.py::SAMLMetadataTest::test_secure_key_configuration
# needs to be fixed in the follow up issue
# https://github.com/openedx/edx-platform/issues/34103
optimizely-sdk<5.0
# Date: 2024-04-30
# lxml>=5.0 introduced breaking changes related to system dependencies
# lxml==5.2.1 introduced new extra so we'll nee to rename lxml --> lxml[html-clean]
# This constraint can be removed once we upgrade to Python 3.11
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35272
lxml<5.0
# This has to be constrained as well because newer versions of edx-i18n-tools need the
# newer version of lxml but that requirement was not made expilict in the 1.6.0 version
# of the package. This can be un-pinned when we're upgrading lxml.
edx-i18n-tools<1.6.0
# xmlsec==1.3.14 breaking tests for all builds, can be removed once a fix is available
xmlsec<1.3.14
# Date: 2018-12-14
# markdown>=3.4.0 has failures due to internal refactorings which causes the tests to fail
# pinning the version untill the issue gets resolved in the package itself
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35271
markdown<3.4.0
# Date: 2024-04-24
# moto==5.0 contains breaking changes. Needs to be updated separately.
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35270
moto<5.0
# path==16.12.0 breaks the unit test collections check
# needs to be investigated and fixed separately
path<16.12.0
# Temporary to Support the python 3.11 Upgrade
backports.zoneinfo;python_version<"3.9" # Newer versions have zoneinfo available in the standard library
# Relevant GitHub Issue: https://github.com/openedx/edx-platform/issues/35126
# Date: 2024-07-16
# We need to upgrade the version of elasticsearch to atleast 7.15 before we can upgrade to Numpy 2.0.0
# Otherwise we see a failure while running the following command:
# export DJANGO_SETTINGS_MODULE=cms.envs.test; python manage.py cms check_reserved_keywords --override_file db_keyword_overrides.yml --report_path reports/reserved_keywords --report_file cms_reserved_keyword_report.csv
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35126
numpy<2.0.0
# django-storages==1.14.4 breaks course imports
# Two lines were added in 1.14.4 that make file_exists_in_storage function always return False,
# as the default value of AWS_S3_FILE_OVERWRITE is True
django-storages<1.14.4
# Date: 2024-01-26
# optimizely-sdk 5.0.0 is breaking following test with segmentation fault
# common/djangoapps/third_party_auth/tests/test_views.py::SAMLMetadataTest::test_secure_key_configuration
# needs to be fixed in the follow up issue
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/34103
optimizely-sdk<5.0
# Date: 2023-09-18
# pinning this version to avoid updates while the library is being developed
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35269
openedx-learning==0.13.1
# Date: 2023-11-29
# Open AI version 1.0.0 dropped support for openai.ChatCompletion which is currently in use in enterprise.
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35268
openai<=0.28.1
# Date: 2024-04-26
# path==16.12.0 breaks the unit test collections check
# needs to be investigated and fixed separately
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35267
path<16.12.0
# Date: 2022-08-03
# pycodestyle==2.9.0 generates false positive error E275.
# Constraint can be removed once the issue https://github.com/PyCQA/pycodestyle/issues/1090 is fixed.
pycodestyle<2.9.0
# Date: 2021-07-12
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/33560
pylint<2.16.0 # greater version failing quality test. Fix them in seperate ticket.
# Date: 2021-08-25
# At the time of writing this comment, we do not know whether py2neo>=2022
# will support our currently-deployed Neo4j version (3.5).
# Feel free to loosen this constraint if/when it is confirmed that a later
# version of py2neo will work with Neo4j 3.5.
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35266
py2neo<2022
# Date: 2020-04-08
# Adding pin to avoid any major upgrade
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35265
pymongo<4.4.1
# Date: 2024-08-06
# social-auth-app-django 5.4.2 introduces a new migration that will not play nicely with large installations. This will touch
# user tables, which are quite large, especially on instances like edx.org.
# We are pinning this until after all the smaller migrations get handled and then we can migrate this all at once.
# Ticket to unpin: https://github.com/edx/edx-arch-experiments/issues/760
# Issue for unpinning: https://github.com/edx/edx-arch-experiments/issues/760
social-auth-app-django<=5.4.1
# Date: 2023-11-05
# urllib3>=2.0.0 conflicts with elastic search && snowflake-connector-python packages
# which require urllib3<2 for now.
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/32222
urllib3<2.0.0
# Date: 2024-04-24
# xmlsec==1.3.14 breaking tests or all builds, can be removed once a fix is available
# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35264
xmlsec<1.3.14