From 5682bc2f3ad3b440a445a83fc6000ff228a0852f Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Mon, 21 Feb 2022 14:09:02 -0500 Subject: [PATCH 01/52] fix: icon on course outline was not showing correctly --- openedx/features/course_experience/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/features/course_experience/utils.py b/openedx/features/course_experience/utils.py index 031e1d1f05..4ac955766f 100644 --- a/openedx/features/course_experience/utils.py +++ b/openedx/features/course_experience/utils.py @@ -51,7 +51,7 @@ def get_course_outline_block_tree(request, course_id, user=None, allow_start_dat is_scored = block.get('has_score', False) and block.get('weight', 1) > 0 # Use a list comprehension to force the recursion over all children, rather than just stopping # at the first child that is scored. - children_scored = any(recurse_mark_scored(child) for child in block.get('children', [])) + children_scored = any(tuple(recurse_mark_scored(child) for child in block.get('children', []))) if is_scored or children_scored: block['scored'] = True return True From 6850c10647890918c58f9eb1fd501cdebf61cf66 Mon Sep 17 00:00:00 2001 From: Dillon Dumesnil Date: Fri, 11 Mar 2022 10:43:53 -0500 Subject: [PATCH 02/52] feat: Updates the Dates Tab to be a static tab --- lms/djangoapps/courseware/tabs.py | 5 ----- lms/djangoapps/courseware/tests/test_tabs.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/lms/djangoapps/courseware/tabs.py b/lms/djangoapps/courseware/tabs.py index 54190fa833..cb2c349f5f 100644 --- a/lms/djangoapps/courseware/tabs.py +++ b/lms/djangoapps/courseware/tabs.py @@ -330,7 +330,6 @@ class DatesTab(EnrolledTab): title = gettext_noop("Dates") priority = 30 view_name = "dates" - is_dynamic = True def __init__(self, tab_dict): def link_func(course, reverse_func): @@ -367,10 +366,6 @@ def get_course_tab_list(user, course): if tab.type == 'static_tab' and tab.course_staff_only and \ not bool(user and has_access(user, 'staff', course, course.id)): continue - # We are phasing this out in https://github.com/openedx/edx-platform/pull/30045/, but need this - # until the backfill course tabs command is completed - if tab.type == 'dates': - continue course_tab_list.append(tab) # Add in any dynamic tabs, i.e. those that are not persisted diff --git a/lms/djangoapps/courseware/tests/test_tabs.py b/lms/djangoapps/courseware/tests/test_tabs.py index 82b1c672ba..b159c20b84 100644 --- a/lms/djangoapps/courseware/tests/test_tabs.py +++ b/lms/djangoapps/courseware/tests/test_tabs.py @@ -390,7 +390,7 @@ class EntranceExamsTabsTestCase(LoginEnrollmentTestCase, ModuleStoreTestCase, Mi milestone ) course_tab_list = get_course_tab_list(self.user, self.course) - assert len(course_tab_list) == 2 + assert len(course_tab_list) == 1 assert course_tab_list[0]['tab_id'] == 'courseware' assert course_tab_list[0]['name'] == 'Entrance Exam' From 9fa79809d8c027deff75877059ace7b900ae371c Mon Sep 17 00:00:00 2001 From: Robert Raposa Date: Wed, 16 Mar 2022 21:17:14 -0400 Subject: [PATCH 03/52] refactor: CookieMonitoringMiddleware moved to edx-django-utils The CookieMonitoringMiddleware and its related script moved to edx-django-utils. ARCHBOM-2054 --- cms/envs/common.py | 11 +- .../scripts/process_cookie_monitoring_logs.py | 212 ------------------ lms/envs/common.py | 15 +- .../user_api/accounts/tests/test_views.py | 12 +- .../core/djangoapps/waffle_utils/testutils.py | 2 +- openedx/core/djangolib/testing/utils.py | 6 +- openedx/core/lib/request_utils.py | 139 ------------ openedx/core/lib/tests/test_request_utils.py | 126 ----------- requirements/edx/base.txt | 8 +- requirements/edx/development.txt | 7 +- requirements/edx/doc.txt | 2 +- requirements/edx/github.in | 3 + requirements/edx/paver.txt | 2 +- requirements/edx/testing.txt | 7 +- scripts/xblock/requirements.txt | 2 +- 15 files changed, 38 insertions(+), 516 deletions(-) delete mode 100644 lms/djangoapps/monitoring/scripts/process_cookie_monitoring_logs.py diff --git a/cms/envs/common.py b/cms/envs/common.py index cd7eb4b9de..e4f050aa17 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -752,16 +752,15 @@ XQUEUE_INTERFACE = { MIDDLEWARE = [ 'openedx.core.lib.x_forwarded_for.middleware.XForwardedForMiddleware', - 'crum.CurrentRequestUserMiddleware', - 'edx_django_utils.monitoring.DeploymentMonitoringMiddleware', - # A newer and safer request cache. + # Resets the request cache. 'edx_django_utils.cache.middleware.RequestCacheMiddleware', - 'edx_django_utils.monitoring.MonitoringMemoryMiddleware', - # Cookie monitoring - 'openedx.core.lib.request_utils.CookieMonitoringMiddleware', + # Various monitoring middleware + 'edx_django_utils.monitoring.CookieMonitoringMiddleware', + 'edx_django_utils.monitoring.DeploymentMonitoringMiddleware', + 'edx_django_utils.monitoring.MonitoringMemoryMiddleware', # Before anything that looks at cookies, especially the session middleware 'openedx.core.djangoapps.cookie_metadata.middleware.CookieNameChange', diff --git a/lms/djangoapps/monitoring/scripts/process_cookie_monitoring_logs.py b/lms/djangoapps/monitoring/scripts/process_cookie_monitoring_logs.py deleted file mode 100644 index 3a05774333..0000000000 --- a/lms/djangoapps/monitoring/scripts/process_cookie_monitoring_logs.py +++ /dev/null @@ -1,212 +0,0 @@ -""" -This script will process logs generated from CookieMonitoringMiddleware. - -Sample usage:: - - python lms/djangoapps/monitoring/scripts/process_cookie_monitoring_logs.py --csv_input large-cookie-logs.csv - -Or for more details:: - - python lms/djangoapps/monitoring/scripts/process_cookie_monitoring_logs.py --help - -""" -import csv -import logging -import re -from dateutil import parser - -import click - -logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.INFO) - - -# Note: Not all Open edX deployments will be affected by the same third-party cookies, -# but it is ok to have some of these cookies go unused. -PARAMETERIZED_COOKIES = [ - (re.compile(r"_gac_UA-(\d|-)+"), "_gac_UA-{id}"), - (re.compile(r"_hjSession_\d+"), "_hjSession_{id}"), - (re.compile(r"_hjSessionUser_\d+"), "_hjSessionUser_{id}"), - (re.compile(r"ab\.storage\.deviceId\..*"), "ab.storage.deviceId.{id}"), - (re.compile(r"ab\.storage\.sessionId\..*"), "ab.storage.deviceId.{id}"), - (re.compile(r"ab\.storage\.userId\..*"), "ab.storage.userId.{id}"), - (re.compile(r"AMCV_\w+%40AdobeOrg"), "AMCV_{id}@AdobeOrg"), - (re.compile(r"amplitude_id_.*"), "amplitude_id_{id}"), - (re.compile(r"mp_\w+_mixpanel"), "mp_{id}_mixpanel"), -] - - -@click.command() -@click.option( - "--csv_input", - help="File name of .csv file with Splunk logs for large cookie headers.", - required=True -) -def main(csv_input): - """ - Reads CSV of large cookie logs and processes and provides summary output. - - Expected CSV format (from Splunk export searching on "BEGIN-COOKIE-SIZES"): - - \b - _raw,_time,index, - ... - - """ - cookie_headers = _load_csv(csv_input) - processed_cookie_headers = process_cookie_headers(cookie_headers) - print_processed_cookies(processed_cookie_headers) - - -def _load_csv(csv_file): - """ - Reads CSV of large cookie data and returns a dict of details. - - Arguments: - csv_file (string): File name for the csv - - Returns a list of dicts containing parsed details for each cookie header log entry. - - """ - with open(csv_file) as file: - csv_data = file.read() - reader = csv.DictReader(csv_data.splitlines()) - - # Regex to match against log messages like the following: - # BEGIN-COOKIE-SIZES(total=3773) user-info: 903, csrftoken: 64, ... END-COOKIE-SIZES - cookie_log_regex = re.compile(r"BEGIN-COOKIE-SIZES\(total=(?P\d+)\)(?P.*)END-COOKIE-SIZES") - # Regex to match against just a single size, like the following: - # csrftoken: 64 - cookie_size_regex = re.compile(r"(?P.*): (?P\d+)") - - cookie_headers = [] - for row in reader: - cookie_header_sizes = {} - - raw_cookie_log = row.get("_raw") - cookie_begin_count = raw_cookie_log.count("BEGIN-COOKIE-SIZES") - if cookie_begin_count == 0: - logging.info("No BEGIN-COOKIE-SIZES delimiter found. Skipping row.") - elif cookie_begin_count > 1: - # Note: this wouldn't parse correctly right now, and it isn't worth coding for. - logging.warning("Multiple cookie entries found in same row. Skipping row.") - continue - match = cookie_log_regex.search(raw_cookie_log) - if not match: - logging.error("Malformed cookie entry. Skipping row.") - continue - - cookie_header_size = int(match.group("total")) - if cookie_header_size == 0: - continue - - cookie_sizes_str = match.group("cookie_sizes").strip() - - cookie_sizes = cookie_sizes_str.split(", ") - for cookie_size in cookie_sizes: - match = cookie_size_regex.search(cookie_size) - if not match: - logging.error(f"Could not parse cookie size from: {cookie_size}") - continue - cookie_header_sizes[match.group("name")] = int(match.group("size")) - - cookie_header_size_computed = max( - 0, sum(len(name) + size + 3 for (name, size) in cookie_header_sizes.items()) - 2 - ) - - cookie_headers.append({ - "datetime": parser.parse(row.get("_time")), - "env": row.get("index"), - "cookie_header_size": cookie_header_size, - "cookie_header_size_computed": cookie_header_size_computed, - "cookie_sizes": cookie_header_sizes, - }) - - return cookie_headers - - -def process_cookie_headers(cookie_headers): - """ - Process the parsed cookie header log entries. - - Arguments: - cookie_headers: a list of dicts containing parsed details. - - Returns a dict of processed cookies. - """ - processed_cookies = {} - for cookie_header in cookie_headers: - for (name, size) in cookie_header["cookie_sizes"].items(): - - # Replace parameterized cookies. For example: - # _hjSessionUser_111111 => _hjSessionUser_{id} - for (regex, replacement_name) in PARAMETERIZED_COOKIES: - if regex.fullmatch(name): - logging.debug(f"Replacing {name} with {replacement_name}.") - name = replacement_name - break - - processed_cookie = processed_cookies.get(name, {}) - - # compute the full size each cookie takes up in the cookie header, including name and delimiters - full_size = len(name) + size + 3 - set_max_attribute(processed_cookie, "max_full_size", full_size) - set_min_attribute(processed_cookie, "min_full_size", full_size) - - set_max_attribute(processed_cookie, "max_size", size) - set_min_attribute(processed_cookie, "min_size", size) - - processed_cookie["count"] = processed_cookie.get("count", 0) + 1 - - # Note: The following details relate to the header, and not to the specific cookie. - # This may give a quick view of cookies associated with the largest header, or the - # header with the most cookies. - - set_max_attribute(processed_cookie, "last_seen", cookie_header["datetime"]) - set_min_attribute(processed_cookie, "first_seen", cookie_header["datetime"]) - - set_max_attribute(processed_cookie, "max_cookie_count", len(cookie_header["cookie_sizes"])) - set_min_attribute(processed_cookie, "min_cookie_count", len(cookie_header["cookie_sizes"])) - set_max_attribute(processed_cookie, "max_header_size", cookie_header["cookie_header_size"]) - set_min_attribute(processed_cookie, "min_header_size", cookie_header["cookie_header_size"]) - # Note: skipping cookie_header_size_calculated unless we see a need for it. - - processed_cookie["envs"] = processed_cookie.get("envs", set()) - processed_cookie["envs"].add(cookie_header["env"]) - - processed_cookies[name] = processed_cookie - - return processed_cookies - - -def set_min_attribute(processed_cookie, key, value): - """ - Sets processed_cookie[key] to the smaller of value and its current value. - """ - processed_cookie[key] = min(value, processed_cookie.get(key, value)) - - -def set_max_attribute(processed_cookie, key, value): - """ - Sets processed_cookie[key] to the larger of value and its current value. - """ - processed_cookie[key] = max(value, processed_cookie.get(key, value)) - - -def print_processed_cookies(processed_cookies): - """ - Output processed cookie information. - """ - sorted_cookie_items = sorted(processed_cookies.items(), key=lambda x: x[1]["max_full_size"], reverse=True) - print("name,max_full_size,min_full_size,max_size,min_size,count," - "last_seen,first_seen,max_cookie_count,min_cookie_count,max_header_size,min_header_size,envs") - for (name, data) in sorted_cookie_items: - print(f'{name},{data["max_full_size"]},{data["min_full_size"]},' - f'{data["max_size"]},{data["min_size"]},{data["count"]},' - f'{data["last_seen"]},{data["first_seen"]},' - f'{data["max_cookie_count"]},{data["min_cookie_count"]},' - f'{data["max_header_size"]},{data["min_header_size"]},' - f'"{",".join(sorted(data["envs"]))}"') - - -if __name__ == "__main__": - main() # pylint: disable=no-value-for-parameter diff --git a/lms/envs/common.py b/lms/envs/common.py index 3602943ca7..5f0f10c02a 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -2057,25 +2057,22 @@ CREDIT_NOTIFICATION_CACHE_TIMEOUT = 5 * 60 * 60 MIDDLEWARE = [ 'openedx.core.lib.x_forwarded_for.middleware.XForwardedForMiddleware', - 'crum.CurrentRequestUserMiddleware', - 'edx_django_utils.monitoring.DeploymentMonitoringMiddleware', - # A newer and safer request cache. + # Resets the request cache. 'edx_django_utils.cache.middleware.RequestCacheMiddleware', - # Generate code ownership attributes. Keep this immediately after RequestCacheMiddleware. + # Various monitoring middleware + 'edx_django_utils.monitoring.CachedCustomMonitoringMiddleware', 'edx_django_utils.monitoring.CodeOwnerMonitoringMiddleware', + 'edx_django_utils.monitoring.CookieMonitoringMiddleware', + 'edx_django_utils.monitoring.DeploymentMonitoringMiddleware', # Before anything that looks at cookies, especially the session middleware 'openedx.core.djangoapps.cookie_metadata.middleware.CookieNameChange', - # Monitoring and logging middleware + # Monitoring and logging for expected and ignored errors 'openedx.core.lib.request_utils.ExpectedErrorMiddleware', - 'edx_django_utils.monitoring.CachedCustomMonitoringMiddleware', - - # Cookie monitoring - 'openedx.core.lib.request_utils.CookieMonitoringMiddleware', 'lms.djangoapps.mobile_api.middleware.AppVersionUpgrade', 'openedx.core.djangoapps.header_control.middleware.HeaderControlMiddleware', diff --git a/openedx/core/djangoapps/user_api/accounts/tests/test_views.py b/openedx/core/djangoapps/user_api/accounts/tests/test_views.py index bd67e80b08..ff078119bd 100644 --- a/openedx/core/djangoapps/user_api/accounts/tests/test_views.py +++ b/openedx/core/djangoapps/user_api/accounts/tests/test_views.py @@ -27,7 +27,7 @@ from openedx.core.djangoapps.user_api.accounts import ACCOUNT_VISIBILITY_PREF_KE from openedx.core.djangoapps.user_api.models import UserPreference from openedx.core.djangoapps.user_api.preferences.api import set_user_preference from openedx.core.djangoapps.waffle_utils.testutils import WAFFLE_TABLES -from openedx.core.djangolib.testing.utils import CacheIsolationTestCase, skip_unless_lms +from openedx.core.djangolib.testing.utils import CacheIsolationTestCase, FilteredQueryCountMixin, skip_unless_lms from openedx.features.name_affirmation_api.utils import get_name_affirmation_service from .. import ALL_USERS_VISIBILITY, CUSTOM_VISIBILITY, PRIVATE_VISIBILITY @@ -195,7 +195,7 @@ class UserAPITestCase(APITestCase): @ddt.ddt @skip_unless_lms -class TestOwnUsernameAPI(CacheIsolationTestCase, UserAPITestCase): +class TestOwnUsernameAPI(FilteredQueryCountMixin, CacheIsolationTestCase, UserAPITestCase): """ Unit tests for the Accounts API. """ @@ -253,13 +253,13 @@ class TestOwnUsernameAPI(CacheIsolationTestCase, UserAPITestCase): {'full': 50, 'small': 10}, clear=True ) -class TestAccountsAPI(CacheIsolationTestCase, UserAPITestCase): +class TestAccountsAPI(FilteredQueryCountMixin, CacheIsolationTestCase, UserAPITestCase): """ Unit tests for the Accounts API. """ ENABLED_CACHES = ['default'] - TOTAL_QUERY_COUNT = 25 + TOTAL_QUERY_COUNT = 24 FULL_RESPONSE_FIELD_COUNT = 30 def setUp(self): @@ -700,7 +700,7 @@ class TestAccountsAPI(CacheIsolationTestCase, UserAPITestCase): assert data['accomplishments_shared'] is False self.client.login(username=self.user.username, password=TEST_PASSWORD) - verify_get_own_information(self._get_num_queries(23)) + verify_get_own_information(self._get_num_queries(22)) # Now make sure that the user can get the same information, even if not active self.user.is_active = False @@ -720,7 +720,7 @@ class TestAccountsAPI(CacheIsolationTestCase, UserAPITestCase): legacy_profile.save() self.client.login(username=self.user.username, password=TEST_PASSWORD) - with self.assertNumQueries(self._get_num_queries(23), table_ignorelist=WAFFLE_TABLES): + with self.assertNumQueries(self._get_num_queries(22), table_ignorelist=WAFFLE_TABLES): response = self.send_get(self.client) for empty_field in ("level_of_education", "gender", "country", "state", "bio",): assert response.data[empty_field] is None diff --git a/openedx/core/djangoapps/waffle_utils/testutils.py b/openedx/core/djangoapps/waffle_utils/testutils.py index d1c8b7a7ad..dbec1d9346 100644 --- a/openedx/core/djangoapps/waffle_utils/testutils.py +++ b/openedx/core/djangoapps/waffle_utils/testutils.py @@ -2,7 +2,7 @@ Test utilities for waffle utilities. """ -# Can be used with FilteredQueryCountMixin.assertNumQueries() to blacklist +# Can be used with FilteredQueryCountMixin.assertNumQueries() to ignore # waffle tables. For example: # QUERY_COUNT_TABLE_IGNORELIST = WAFFLE_TABLES # with self.assertNumQueries(6, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST): diff --git a/openedx/core/djangolib/testing/utils.py b/openedx/core/djangolib/testing/utils.py index e19ffa256f..2aacd444d3 100644 --- a/openedx/core/djangolib/testing/utils.py +++ b/openedx/core/djangolib/testing/utils.py @@ -170,7 +170,7 @@ class _AssertNumQueriesContext(CaptureQueriesContext): def __exit__(self, exc_type, exc_value, traceback): def is_unfiltered_query(query): """ - Returns True if the query does not contain a blacklisted table, and + Returns True if the query does not contain a ignorelisted table, and False otherwise. Note: This is a simple naive implementation that makes no attempt @@ -201,7 +201,7 @@ class _AssertNumQueriesContext(CaptureQueriesContext): class FilteredQueryCountMixin: """ Mixin to add to any subclass of Django's TestCase that replaces - assertNumQueries with one that accepts a blacklist of tables to filter out + assertNumQueries with one that accepts a ignorelist of tables to filter out of the count. """ def assertNumQueries(self, num, func=None, table_ignorelist=None, *args, **kwargs): # lint-amnesty, pylint: disable=keyword-arg-before-vararg @@ -210,6 +210,8 @@ class FilteredQueryCountMixin: the addition of the following argument: table_ignorelist (List): A list of table names to filter out of the set of queries that get counted. + + Important: TestCase must include FilteredQueryCountMixin for this to work. """ using = kwargs.pop("using", DEFAULT_DB_ALIAS) conn = connections[using] diff --git a/openedx/core/lib/request_utils.py b/openedx/core/lib/request_utils.py index f14da167af..6a6bb39f6e 100644 --- a/openedx/core/lib/request_utils.py +++ b/openedx/core/lib/request_utils.py @@ -1,7 +1,6 @@ """ Utility functions related to HTTP requests """ import logging -import random import re from urllib.parse import urlparse @@ -90,144 +89,6 @@ def course_id_from_url(url): return None -class CookieMonitoringMiddleware: - """ - Middleware for monitoring the size and growth of all our cookies, to see if - we're running into browser limits. - """ - def __init__(self, get_response): - self.get_response = get_response - - def __call__(self, request): - # Monitor at request-time to skip any cookies that may be added during the request. - log_message = None - try: - log_message = self.get_log_message_and_monitor_cookies(request) - except BaseException: - log.exception("Unexpected error logging and monitoring cookies.") - - response = self.get_response(request) - - # Delay logging until response-time so that the user id can be included in the log message. - if log_message: - log.info(log_message) - - return response - - def get_log_message_and_monitor_cookies(self, request): - """ - Add logging and custom attributes for monitoring cookie sizes. - - Don't log contents of cookies because that might cause a security issue. - We just want to see if any cookies are growing out of control. - - Useful NRQL Queries: - - # Always available - SELECT * FROM Transaction WHERE cookies.header.size > 6000 - - Attributes that are added by this middleware: - - For all requests: - - cookies.header.size: The total size in bytes of the cookie header - - If COOKIE_HEADER_SIZE_LOGGING_THRESHOLD is reached: - - cookies.header.size.computed - - Related Settings (see annotations for details): - - - COOKIE_HEADER_SIZE_LOGGING_THRESHOLD - - COOKIE_SAMPLING_REQUEST_COUNT - - Returns: The message to be logged. This is returned, rather than directly - logged, so that it can be processed at request time (before any cookies - may be changed server-side), but logged at response time, once the user - id is available for authenticated calls. - - """ - - raw_header_cookie = request.META.get('HTTP_COOKIE', '') - cookie_header_size = len(raw_header_cookie.encode('utf-8')) - # .. custom_attribute_name: cookies.header.size - # .. custom_attribute_description: The total size in bytes of the cookie header. - set_custom_attribute('cookies.header.size', cookie_header_size) - - if cookie_header_size == 0: - return - - if corrupt_cookie_count := raw_header_cookie.count('Cookie: '): - # .. custom_attribute_name: cookies.header.corrupt_count - # .. custom_attribute_description: The attribute will only appear for potentially corrupt cookie headers, - # where "Cookie: " is found in the header. If this custom attribute is seen on the same - # requests where other mysterious cookie problems are occurring, this may help troubleshoot. - # See https://openedx.atlassian.net/browse/CR-4614 for more details. - # Also see cookies.header.corrupt_key_count - set_custom_attribute('cookies.header.corrupt_count', corrupt_cookie_count) - # .. custom_attribute_name: cookies.header.corrupt_key_count - # .. custom_attribute_description: The attribute will only appear for potentially corrupt cookie headers, - # where "Cookie: " is found in some of the cookie keys. If this custom attribute is seen on the same - # requests where other mysterious cookie problems are occurring, this may help troubleshoot. - # See https://openedx.atlassian.net/browse/CR-4614 for more details. - # Also see cookies.header.corrupt_count. - set_custom_attribute( - 'cookies.header.corrupt_key_count', - sum(1 for key in request.COOKIES.keys() if 'Cookie: ' in key) - ) - - # .. setting_name: COOKIE_HEADER_SIZE_LOGGING_THRESHOLD - # .. setting_default: None - # .. setting_description: The minimum size for the full cookie header to log a list of cookie names and sizes. - # Should be set to a relatively high threshold (suggested 9-10K) to avoid flooding the logs. - logging_threshold = getattr(settings, "COOKIE_HEADER_SIZE_LOGGING_THRESHOLD", None) - - if not logging_threshold: - return - - is_large_cookie_header_detected = cookie_header_size >= logging_threshold - if not is_large_cookie_header_detected: - # .. setting_name: COOKIE_SAMPLING_REQUEST_COUNT - # .. setting_default: None - # .. setting_description: This setting enables sampling cookie header logging for cookie headers smaller - # than COOKIE_HEADER_SIZE_LOGGING_THRESHOLD. The cookie header logging will happen randomly for each - # request with a chance of 1 in COOKIE_SAMPLING_REQUEST_COUNT. For example, to see approximately one - # sampled log message every 10 minutes, set COOKIE_SAMPLING_REQUEST_COUNT to the average number of - # requests in 10 minutes. - # .. setting_warning: This setting requires COOKIE_HEADER_SIZE_LOGGING_THRESHOLD to be enabled to take - # effect. - sampling_request_count = getattr(settings, "COOKIE_SAMPLING_REQUEST_COUNT", None) - - # if the cookie header size is lower than the threshold, skip logging unless configured to do - # random sampling and we choose the lucky number (in this case, 1). - if not sampling_request_count or random.randint(1, sampling_request_count) > 1: - return - - # The computed header size can be used to double check that there aren't large cookies that are - # duplicates in the original header (from different domains) that aren't being accounted for. - cookies_header_size_computed = max( - 0, sum(len(name) + len(value) + 3 for (name, value) in request.COOKIES.items()) - 2 - ) - - # .. custom_attribute_name: cookies.header.size.computed - # .. custom_attribute_description: The computed total size in bytes of the cookie header, based on the - # cookies found in request.COOKIES. This value will only be captured for cookie headers larger than - # COOKIE_HEADER_SIZE_LOGGING_THRESHOLD. The value can be used to double check that there aren't large - # cookies that are duplicates in the cookie header (from different domains) that aren't being accounted - # for. - set_custom_attribute('cookies.header.size.computed', cookies_header_size_computed) - - # Sort starting with largest cookies - sorted_cookie_items = sorted(request.COOKIES.items(), key=lambda x: len(x[1]), reverse=True) - sizes = ', '.join(f"{name}: {len(value)}" for (name, value) in sorted_cookie_items) - if is_large_cookie_header_detected: - log_prefix = f"Large (>= {logging_threshold}) cookie header detected." - else: - log_prefix = f"Sampled small (< {logging_threshold}) cookie header." - log_message = f"{log_prefix} BEGIN-COOKIE-SIZES(total={cookie_header_size}) {sizes} END-COOKIE-SIZES" - return log_message - - def expected_error_exception_handler(exc, context): """ Replacement for DRF's default exception handler to enable observing expected errors. diff --git a/openedx/core/lib/tests/test_request_utils.py b/openedx/core/lib/tests/test_request_utils.py index f918470258..390d5ae3c8 100644 --- a/openedx/core/lib/tests/test_request_utils.py +++ b/openedx/core/lib/tests/test_request_utils.py @@ -12,7 +12,6 @@ from django.test.utils import override_settings from edx_django_utils.cache import RequestCache from openedx.core.lib.request_utils import ( - CookieMonitoringMiddleware, ExpectedErrorMiddleware, _get_expected_error_settings_dict, clear_cached_expected_error_settings, @@ -103,131 +102,6 @@ class RequestUtilTestCase(unittest.TestCase): assert course_id.run == run -@ddt.ddt -class CookieMonitoringMiddlewareTestCase(unittest.TestCase): - """ - Tests for CookieMonitoringMiddleware. - """ - def setUp(self): - super().setUp() - self.mock_response = Mock() - - @patch('openedx.core.lib.request_utils.log', autospec=True) - @patch("openedx.core.lib.request_utils.set_custom_attribute") - @ddt.data( - (None, None), # logging threshold not defined - (5, None), # logging threshold too high - (5, 9999999999999999999), # logging threshold too high, and random sampling impossibly unlikely - ) - @ddt.unpack - def test_cookie_monitoring_with_no_logging( - self, logging_threshold, sampling_request_count, mock_set_custom_attribute, mock_logger - ): - middleware = CookieMonitoringMiddleware(self.mock_response) - cookies_dict = {'a': 'y'} - - with override_settings(COOKIE_HEADER_SIZE_LOGGING_THRESHOLD=logging_threshold): - with override_settings(COOKIE_SAMPLING_REQUEST_COUNT=sampling_request_count): - middleware(self.get_mock_request(cookies_dict)) - - # expect monitoring of header size for all requests - mock_set_custom_attribute.assert_called_once_with('cookies.header.size', 3) - # cookie logging was not enabled, so nothing should be logged - mock_logger.info.assert_not_called() - - @override_settings(COOKIE_HEADER_SIZE_LOGGING_THRESHOLD=None) - @override_settings(COOKIE_SAMPLING_REQUEST_COUNT=None) - @patch("openedx.core.lib.request_utils.set_custom_attribute") - @ddt.data( - # A corrupt cookie header contains "Cookie: ". - ('corruptCookie: normal-cookie=value', 1, 1), - ('corrupt1Cookie: normal-cookie1=value1;corrupt2Cookie: normal-cookie2=value2', 2, 2), - ('corrupt=Cookie: value', 1, 0), - ) - @ddt.unpack - def test_cookie_header_corrupt_monitoring( - self, corrupt_cookie_header, expected_corrupt_count, expected_corrupt_key_count, mock_set_custom_attribute - ): - middleware = CookieMonitoringMiddleware(self.mock_response) - request = RequestFactory().request() - request.META['HTTP_COOKIE'] = corrupt_cookie_header - - middleware(request) - - mock_set_custom_attribute.assert_has_calls([ - call('cookies.header.size', len(request.META['HTTP_COOKIE'])), - call('cookies.header.corrupt_count', expected_corrupt_count), - call('cookies.header.corrupt_key_count', expected_corrupt_key_count), - ]) - - @override_settings(COOKIE_HEADER_SIZE_LOGGING_THRESHOLD=1) - @patch('openedx.core.lib.request_utils.log', autospec=True) - @patch("openedx.core.lib.request_utils.set_custom_attribute") - def test_log_cookie_with_threshold_met(self, mock_set_custom_attribute, mock_logger): - middleware = CookieMonitoringMiddleware(self.mock_response) - cookies_dict = { - "a": "yy", - "b": "xxx", - "c": "z", - } - - middleware(self.get_mock_request(cookies_dict)) - - mock_set_custom_attribute.assert_has_calls([ - call('cookies.header.size', 16), - call('cookies.header.size.computed', 16) - ]) - mock_logger.info.assert_called_once_with( - "Large (>= 1) cookie header detected. BEGIN-COOKIE-SIZES(total=16) b: 3, a: 2, c: 1 END-COOKIE-SIZES" - ) - - @override_settings(COOKIE_HEADER_SIZE_LOGGING_THRESHOLD=9999) - @override_settings(COOKIE_SAMPLING_REQUEST_COUNT=1) - @patch('openedx.core.lib.request_utils.log', autospec=True) - @patch("openedx.core.lib.request_utils.set_custom_attribute") - def test_log_cookie_with_sampling(self, mock_set_custom_attribute, mock_logger): - middleware = CookieMonitoringMiddleware(self.mock_response) - cookies_dict = { - "a": "yy", - "b": "xxx", - "c": "z", - } - - middleware(self.get_mock_request(cookies_dict)) - - mock_set_custom_attribute.assert_has_calls([ - call('cookies.header.size', 16), - call('cookies.header.size.computed', 16) - ]) - mock_logger.info.assert_called_once_with( - "Sampled small (< 9999) cookie header. BEGIN-COOKIE-SIZES(total=16) b: 3, a: 2, c: 1 END-COOKIE-SIZES" - ) - - @override_settings(COOKIE_HEADER_SIZE_LOGGING_THRESHOLD=9999) - @override_settings(COOKIE_SAMPLING_REQUEST_COUNT=1) - @patch('openedx.core.lib.request_utils.log', autospec=True) - @patch("openedx.core.lib.request_utils.set_custom_attribute") - def test_empty_cookie_header_skips_sampling(self, mock_set_custom_attribute, mock_logger): - middleware = CookieMonitoringMiddleware(self.mock_response) - cookies_dict = {} - - middleware(self.get_mock_request(cookies_dict)) - - mock_set_custom_attribute.assert_has_calls([ - call('cookies.header.size', 0), - ]) - mock_logger.info.assert_not_called() - - def get_mock_request(self, cookies_dict): - """ - Return mock request with the provided cookies in the header. - """ - factory = RequestFactory() - for name, value in cookies_dict.items(): - factory.cookies[name] = value - return factory.request() - - class TestGetExpectedErrorSettingsDict(unittest.TestCase): """ Tests for processing issues in _get_expected_error_settings_dict() diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 9d6208fe20..7b8e73a345 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -338,7 +338,7 @@ django-sekizai==3.0.1 # via # -r requirements/edx/base.in # django-wiki -django-ses==2.6.0 +django-ses==2.6.1 # via -r requirements/edx/base.in django-simple-history==3.0.0 # via @@ -428,9 +428,10 @@ edx-django-release-util==1.2.0 # via -r requirements/edx/base.in edx-django-sites-extensions==4.0.0 # via -r requirements/edx/base.in -edx-django-utils==4.5.0 +edx-django-utils==4.6.0 # via # -r requirements/edx/base.in + # -r requirements/edx/github.in # django-config-models # edx-drf-extensions # edx-enterprise @@ -552,7 +553,6 @@ fs-s3fs==0.1.8 # django-pyfs future==0.18.2 # via - # django-ses # edx-celeryutils # pyjwkest geoip2==4.5.0 @@ -1031,7 +1031,7 @@ uritemplate==4.1.1 # via # coreapi # drf-yasg -urllib3==1.26.8 +urllib3==1.26.9 # via # -r requirements/edx/paver.txt # elasticsearch diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 5df20f13fb..ef560608d6 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -434,7 +434,7 @@ django-sekizai==3.0.1 # via # -r requirements/edx/testing.txt # django-wiki -django-ses==2.6.0 +django-ses==2.6.1 # via -r requirements/edx/testing.txt django-simple-history==3.0.0 # via @@ -535,7 +535,7 @@ edx-django-release-util==1.2.0 # via -r requirements/edx/testing.txt edx-django-sites-extensions==4.0.0 # via -r requirements/edx/testing.txt -edx-django-utils==4.5.0 +edx-django-utils==4.6.0 # via # -r requirements/edx/testing.txt # django-config-models @@ -699,7 +699,6 @@ fs-s3fs==0.1.8 future==0.18.2 # via # -r requirements/edx/testing.txt - # django-ses # edx-celeryutils # pyjwkest geoip2==4.5.0 @@ -1497,7 +1496,7 @@ uritemplate==4.1.1 # -r requirements/edx/testing.txt # coreapi # drf-yasg -urllib3==1.26.8 +urllib3==1.26.9 # via # -r requirements/edx/testing.txt # elasticsearch diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 3000afd3a8..056f490660 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -84,7 +84,7 @@ stevedore==3.5.0 # via code-annotations text-unidecode==1.3 # via python-slugify -urllib3==1.26.8 +urllib3==1.26.9 # via requests zipp==3.7.0 # via importlib-metadata diff --git a/requirements/edx/github.in b/requirements/edx/github.in index 54d08e572a..5e126a4d56 100644 --- a/requirements/edx/github.in +++ b/requirements/edx/github.in @@ -75,3 +75,6 @@ git+https://github.com/edx/django-require.git@0c54adb167142383b26ea6b3edecc32118 git+https://github.com/open-craft/xblock-poll@v1.12.0#egg=xblock-poll==1.12.0 git+https://github.com/edx-solutions/xblock-drag-and-drop-v2@v2.3.5#egg=xblock-drag-and-drop-v2==2.3.5 + +# Temporary for testing edx-django-utils upgrade +git+https://github.com/edx/edx-django-utils.git@robrap/ARCHBOM-2054-move-cookie-monitoring-middleware#egg=edx_django_utils diff --git a/requirements/edx/paver.txt b/requirements/edx/paver.txt index cf24727b36..d71382c9e5 100644 --- a/requirements/edx/paver.txt +++ b/requirements/edx/paver.txt @@ -46,7 +46,7 @@ stevedore==3.5.0 # via # -r requirements/edx/paver.in # edx-opaque-keys -urllib3==1.26.8 +urllib3==1.26.9 # via requests watchdog==2.1.6 # via -r requirements/edx/paver.in diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 782b589874..f2ae475e9b 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -420,7 +420,7 @@ django-sekizai==3.0.1 # via # -r requirements/edx/base.txt # django-wiki -django-ses==2.6.0 +django-ses==2.6.1 # via -r requirements/edx/base.txt django-simple-history==3.0.0 # via @@ -519,7 +519,7 @@ edx-django-release-util==1.2.0 # via -r requirements/edx/base.txt edx-django-sites-extensions==4.0.0 # via -r requirements/edx/base.txt -edx-django-utils==4.5.0 +edx-django-utils==4.6.0 # via # -r requirements/edx/base.txt # django-config-models @@ -675,7 +675,6 @@ fs-s3fs==0.1.8 future==0.18.2 # via # -r requirements/edx/base.txt - # django-ses # edx-celeryutils # pyjwkest geoip2==4.5.0 @@ -1384,7 +1383,7 @@ uritemplate==4.1.1 # -r requirements/edx/base.txt # coreapi # drf-yasg -urllib3==1.26.8 +urllib3==1.26.9 # via # -r requirements/edx/base.txt # elasticsearch diff --git a/scripts/xblock/requirements.txt b/scripts/xblock/requirements.txt index 77e8078094..f715aba5af 100644 --- a/scripts/xblock/requirements.txt +++ b/scripts/xblock/requirements.txt @@ -12,5 +12,5 @@ idna==3.3 # via requests requests==2.27.1 # via -r scripts/xblock/requirements.in -urllib3==1.26.8 +urllib3==1.26.9 # via requests From 776c3ab9c0b55c500f814639b63f1fce789bf57c Mon Sep 17 00:00:00 2001 From: Robert Raposa Date: Thu, 17 Mar 2022 11:53:49 -0400 Subject: [PATCH 04/52] docs: warn about test order issue Some registration tests can fail if run in a particular order. This PR just adds a warning so engineers don't mistakenly think they caused an issue. Also see VAN-900 for more details on how to reproduce. --- .../user_authn/views/tests/test_register.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/openedx/core/djangoapps/user_authn/views/tests/test_register.py b/openedx/core/djangoapps/user_authn/views/tests/test_register.py index 1d3191db8d..19b3dc51dc 100644 --- a/openedx/core/djangoapps/user_authn/views/tests/test_register.py +++ b/openedx/core/djangoapps/user_authn/views/tests/test_register.py @@ -934,6 +934,9 @@ class RegistrationViewTestV1( ) def test_register_form_year_of_birth(self): + # WARNING: This test may fail locally due to a test order issue. If it passes on Github, + # but is failing for you locally, you probably did not cause the problem if you + # are not working on registration. See VAN-900 for more details. this_year = datetime.now(UTC).year year_options = ( [ @@ -1320,6 +1323,9 @@ class RegistrationViewTestV1( REGISTRATION_EXTENSION_FORM='openedx.core.djangoapps.user_api.tests.test_helpers.TestCaseForm', ) def test_field_order(self): + # WARNING: This test may fail locally due to a test order issue. If it passes on Github, + # but is failing for you locally, you probably did not cause the problem if you + # are not working on registration. See VAN-900 for more details. response = self.client.get(self.url) self.assertHttpOK(response) @@ -1382,6 +1388,9 @@ class RegistrationViewTestV1( ], ) def test_field_order_override(self): + # WARNING: This test may fail locally due to a test order issue. If it passes on Github, + # but is failing for you locally, you probably did not cause the problem if you + # are not working on registration. See VAN-900 for more details. response = self.client.get(self.url) self.assertHttpOK(response) @@ -1423,6 +1432,9 @@ class RegistrationViewTestV1( ], ) def test_field_order_invalid_override(self): + # WARNING: This test may fail locally due to a test order issue. If it passes on Github, + # but is failing for you locally, you probably did not cause the problem if you + # are not working on registration. See VAN-900 for more details. response = self.client.get(self.url) self.assertHttpOK(response) @@ -2002,6 +2014,9 @@ class RegistrationViewTestV2(RegistrationViewTestV1): ], ) def test_field_order_invalid_override(self): + # WARNING: This test may fail locally due to a test order issue. If it passes on Github, + # but is failing for you locally, you probably did not cause the problem if you + # are not working on registration. See VAN-900 for more details. response = self.client.get(self.url) self.assertHttpOK(response) @@ -2066,6 +2081,9 @@ class RegistrationViewTestV2(RegistrationViewTestV1): ], ) def test_field_order_override(self): + # WARNING: This test may fail locally due to a test order issue. If it passes on Github, + # but is failing for you locally, you probably did not cause the problem if you + # are not working on registration. See VAN-900 for more details. response = self.client.get(self.url) self.assertHttpOK(response) @@ -2094,6 +2112,9 @@ class RegistrationViewTestV2(RegistrationViewTestV1): REGISTRATION_EXTENSION_FORM='openedx.core.djangoapps.user_api.tests.test_helpers.TestCaseForm', ) def test_field_order(self): + # WARNING: This test may fail locally due to a test order issue. If it passes on Github, + # but is failing for you locally, you probably did not cause the problem if you + # are not working on registration. See VAN-900 for more details. response = self.client.get(self.url) self.assertHttpOK(response) From 699afeb731e2e49c20120d6a4f1032e4df5d578a Mon Sep 17 00:00:00 2001 From: Abdurrahman Asad <51022010+A-ASAD@users.noreply.github.com> Date: Fri, 18 Mar 2022 19:16:15 +0500 Subject: [PATCH 05/52] fix: flagged post banner not showing to staff (#30034) fix: flagged post banner not showing to staff --- lms/djangoapps/discussion/rest_api/serializers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lms/djangoapps/discussion/rest_api/serializers.py b/lms/djangoapps/discussion/rest_api/serializers.py index 03675d5fb9..d890046474 100644 --- a/lms/djangoapps/discussion/rest_api/serializers.py +++ b/lms/djangoapps/discussion/rest_api/serializers.py @@ -237,7 +237,11 @@ class _ContentSerializer(serializers.Serializer): Returns a boolean indicating whether the requester has flagged the content as abusive. """ - return self.context["cc_requester"]["id"] in obj.get("abuse_flaggers", []) + total_abuse_flaggers = len(obj.get("abuse_flaggers", [])) + return ( + self.context["is_requester_privileged"] and total_abuse_flaggers > 0 or + self.context["cc_requester"]["id"] in obj.get("abuse_flaggers", []) + ) def get_voted(self, obj): """ From c3bc68abc1641e45c45d5160c12c0107fb055ba0 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Fri, 18 Mar 2022 15:31:27 +0000 Subject: [PATCH 06/52] feat: Add monitoring for X-Forwarded-For header length (#30090) --- .../core/lib/x_forwarded_for/middleware.py | 7 ++ .../lib/x_forwarded_for/tests/__init__.py | 0 .../x_forwarded_for/tests/test_middleware.py | 75 +++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 openedx/core/lib/x_forwarded_for/tests/__init__.py create mode 100644 openedx/core/lib/x_forwarded_for/tests/test_middleware.py diff --git a/openedx/core/lib/x_forwarded_for/middleware.py b/openedx/core/lib/x_forwarded_for/middleware.py index 98a8bf7bf9..7d4447a8af 100644 --- a/openedx/core/lib/x_forwarded_for/middleware.py +++ b/openedx/core/lib/x_forwarded_for/middleware.py @@ -3,7 +3,9 @@ Middleware to use the X-Forwarded-For header as the request IP. Updated the libray to use HTTP_HOST and X-Forwarded-Port as SERVER_NAME and SERVER_PORT. """ + from django.utils.deprecation import MiddlewareMixin +from edx_django_utils.monitoring import set_custom_attribute class XForwardedForMiddleware(MiddlewareMixin): @@ -19,6 +21,11 @@ class XForwardedForMiddleware(MiddlewareMixin): Process the given request, update the value of REMOTE_ADDR, SERVER_NAME and SERVER_PORT based on X-Forwarded-For, HTTP_HOST and X-Forwarded-Port headers """ + # Give some observability into X-Forwarded-For length. Useful + # for monitoring in case of unexpected changes, etc. + xff = request.META.get('HTTP_X_FORWARDED_FOR') + xff_len = xff.count(',') + 1 if xff else 0 + set_custom_attribute('header.x-forwarded-for.count', xff_len) for field, header in [("HTTP_X_FORWARDED_FOR", "REMOTE_ADDR"), ("HTTP_HOST", "SERVER_NAME"), ("HTTP_X_FORWARDED_PORT", "SERVER_PORT")]: diff --git a/openedx/core/lib/x_forwarded_for/tests/__init__.py b/openedx/core/lib/x_forwarded_for/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openedx/core/lib/x_forwarded_for/tests/test_middleware.py b/openedx/core/lib/x_forwarded_for/tests/test_middleware.py new file mode 100644 index 0000000000..52d1489eb6 --- /dev/null +++ b/openedx/core/lib/x_forwarded_for/tests/test_middleware.py @@ -0,0 +1,75 @@ +""" +Tests for XForwardedForMiddleware. +""" + +from unittest.mock import call, patch + +import ddt +from django.test import TestCase +from django.test.client import RequestFactory + +from openedx.core.lib.x_forwarded_for.middleware import XForwardedForMiddleware + + +@ddt.ddt +class TestXForwardedForMiddleware(TestCase): + """Tests for middleware's overrides.""" + + @ddt.unpack + @ddt.data( + # With no added headers, just see the test server's defaults. + ( + {}, + { + 'SERVER_NAME': 'testserver', + 'SERVER_PORT': '80', + 'REMOTE_ADDR': '127.0.0.1', + }, + ), + # With headers supplied by the request (Host) and a proxy + # (X-Forwarded-Port), see the name and port overridden. + ( + { + 'HTTP_HOST': 'example.com', + 'HTTP_X_FORWARDED_PORT': '443', + }, + { + 'SERVER_NAME': 'example.com', + 'SERVER_PORT': '443', + }, + ), + + # REMOTE_ADDR can also be overridden + ( + {'HTTP_X_FORWARDED_FOR': '7.8.9.0, 1.2.3.4'}, + { + 'REMOTE_ADDR': '7.8.9.0', + }, + ), + ) + def test_overrides(self, add_meta, expected_meta_include): + """ + Test that parts of request.META can be overridden by HTTP headers. + """ + request = RequestFactory().get('/somewhere') + request.META.update(add_meta) + + XForwardedForMiddleware().process_request(request) + + assert request.META.items() >= expected_meta_include.items() + + @ddt.unpack + @ddt.data( + (None, 0), + ('1.2.3.4', 1), + ('7.8.9.0, 1.2.3.4, 5.5.5.5', 3), + ) + @patch("openedx.core.lib.x_forwarded_for.middleware.set_custom_attribute") + def test_xff_metrics(self, xff, expected_count, mock_set_custom_attribute): + request = RequestFactory().get('/somewhere') + if xff is not None: + request.META['HTTP_X_FORWARDED_FOR'] = xff + + XForwardedForMiddleware().process_request(request) + + mock_set_custom_attribute.assert_has_calls([call('header.x-forwarded-for.count', expected_count)]) From 35df2723d8d84e5df3f8677dd717a25c5772316d Mon Sep 17 00:00:00 2001 From: Michael Terry Date: Fri, 18 Mar 2022 11:06:55 -0400 Subject: [PATCH 07/52] feat: add a config model for the backfill_course_tabs command - Adds a BackfillCourseTabsConfig model to manage the arguments to that command - Adds batching arguments using that model - Adds some extra logging for the failed courses --- cms/djangoapps/contentstore/admin.py | 3 +- .../commands/backfill_course_tabs.py | 21 +++++++++++-- .../tests/test_backfill_course_tabs.py | 31 ++++++++++++++++--- .../0007_backfillcoursetabsconfig.py | 31 +++++++++++++++++++ cms/djangoapps/contentstore/models.py | 23 +++++++++++++- 5 files changed, 100 insertions(+), 9 deletions(-) create mode 100644 cms/djangoapps/contentstore/migrations/0007_backfillcoursetabsconfig.py diff --git a/cms/djangoapps/contentstore/admin.py b/cms/djangoapps/contentstore/admin.py index cd70873b4b..12f9dfcf6f 100644 --- a/cms/djangoapps/contentstore/admin.py +++ b/cms/djangoapps/contentstore/admin.py @@ -10,7 +10,7 @@ from django.contrib.admin.helpers import ACTION_CHECKBOX_NAME from django.utils.translation import gettext as _ from edx_django_utils.admin.mixins import ReadOnlyAdminMixin -from cms.djangoapps.contentstore.models import VideoUploadConfig +from cms.djangoapps.contentstore.models import BackfillCourseTabsConfig, VideoUploadConfig from cms.djangoapps.contentstore.outlines_regenerate import CourseOutlineRegenerate from openedx.core.djangoapps.content.learning_sequences.api import key_supports_outlines @@ -78,5 +78,6 @@ class CourseOutlineRegenerateAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): return super().changelist_view(request, extra_context) +admin.site.register(BackfillCourseTabsConfig, ConfigurationModelAdmin) admin.site.register(VideoUploadConfig, ConfigurationModelAdmin) admin.site.register(CourseOutlineRegenerate, CourseOutlineRegenerateAdmin) diff --git a/cms/djangoapps/contentstore/management/commands/backfill_course_tabs.py b/cms/djangoapps/contentstore/management/commands/backfill_course_tabs.py index bcbea418f2..878a8dabaa 100644 --- a/cms/djangoapps/contentstore/management/commands/backfill_course_tabs.py +++ b/cms/djangoapps/contentstore/management/commands/backfill_course_tabs.py @@ -12,11 +12,12 @@ Search for the error message to detect any issues. import logging from django.core.management.base import BaseCommand - from xmodule.tabs import CourseTabList from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore +from cms.djangoapps.contentstore.models import BackfillCourseTabsConfig + logger = logging.getLogger(__name__) @@ -37,12 +38,19 @@ class Command(BaseCommand): if there are any new default course tabs. Else, makes no updates. """ store = modulestore() - course_keys = sorted( + all_course_keys = sorted( (course.id for course in store.get_course_summaries()), key=str # Different types of CourseKeys can't be compared without this. ) - logger.info(f'{len(course_keys)} courses read from modulestore.') + config = BackfillCourseTabsConfig.current() + start = config.start_index if config.enabled and config.start_index >= 0 else 0 + end = (start + config.count) if config.enabled and config.count > 0 else len(all_course_keys) + course_keys = all_course_keys[start:end] + + logger.info(f'{len(all_course_keys)} courses read from modulestore. Processing {start} to {end}.') + + error_keys = [] for course_key in course_keys: try: course = store.get_course(course_key, depth=1) @@ -59,3 +67,10 @@ class Command(BaseCommand): except Exception as err: # pylint: disable=broad-except logger.exception(err) logger.error(f'Course {course_key} encountered an Exception while trying to update.') + error_keys.append(course_key) + + if error_keys: + msg = 'The following courses encountered errors and were not updated:\n' + for error_key in error_keys: + msg += f' - {error_key}\n' + logger.info(msg) diff --git a/cms/djangoapps/contentstore/management/commands/tests/test_backfill_course_tabs.py b/cms/djangoapps/contentstore/management/commands/tests/test_backfill_course_tabs.py index 322f8f5064..258f00a133 100644 --- a/cms/djangoapps/contentstore/management/commands/tests/test_backfill_course_tabs.py +++ b/cms/djangoapps/contentstore/management/commands/tests/test_backfill_course_tabs.py @@ -3,13 +3,16 @@ Tests for `backfill_course_outlines` Studio (cms) management command. """ from unittest import mock +import ddt from django.core.management import call_command - from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory +from cms.djangoapps.contentstore.models import BackfillCourseTabsConfig + +@ddt.ddt class BackfillCourseTabsTest(ModuleStoreTestCase): """ Test `backfill_course_tabs` @@ -75,7 +78,7 @@ class BackfillCourseTabsTest(ModuleStoreTestCase): assert len(course.tabs) == 7 assert 'dates' in {tab.type for tab in course.tabs} assert 'progress' in {tab.type for tab in course.tabs} - mock_logger.info.assert_any_call('4 courses read from modulestore.') + mock_logger.info.assert_any_call('4 courses read from modulestore. Processing 0 to 4.') mock_logger.info.assert_any_call(f'Updating tabs for {course.id}.') mock_logger.info.assert_any_call(f'Successfully updated tabs for {course.id}.') assert mock_logger.info.call_count == 3 @@ -109,7 +112,7 @@ class BackfillCourseTabsTest(ModuleStoreTestCase): assert len(course_2.tabs) == 7 assert 'dates' in {tab.type for tab in course_1.tabs} assert 'progress' in {tab.type for tab in course_2.tabs} - mock_logger.info.assert_any_call('2 courses read from modulestore.') + mock_logger.info.assert_any_call('2 courses read from modulestore. Processing 0 to 2.') mock_logger.info.assert_any_call(f'Updating tabs for {course_1.id}.') mock_logger.info.assert_any_call(f'Successfully updated tabs for {course_1.id}.') mock_logger.info.assert_any_call(f'Updating tabs for {course_2.id}.') @@ -149,9 +152,29 @@ class BackfillCourseTabsTest(ModuleStoreTestCase): # Course wasn't updated due to the ValueError assert error_course_tabs_before == error_course_tabs_after - mock_logger.info.assert_any_call('2 courses read from modulestore.') + mock_logger.info.assert_any_call('2 courses read from modulestore. Processing 0 to 2.') mock_logger.info.assert_any_call(f'Successfully updated tabs for {updated_course.id}.') mock_logger.exception.assert_called() mock_logger.error.assert_called_once_with( f'Course {error_course.id} encountered an Exception while trying to update.' ) + + @ddt.data( + (1, 2, [False, True, True, False]), + (1, 0, [False, True, True, True]), + (-1, -1, [True, True, True, True]), + ) + @ddt.unpack + def test_arguments_batching(self, start, count, expected_tabs_modified): + courses = CourseFactory.create_batch(4) + for course in courses: + course.tabs = [tab for tab in course.tabs if tab.type in ('course_info', 'courseware')] + course = self.update_course(course, ModuleStoreEnum.UserID.test) + assert len(course.tabs) == 2 + + BackfillCourseTabsConfig.objects.create(enabled=True, start_index=start, count=count) + call_command('backfill_course_tabs') + + for i, course in enumerate(courses): + course = self.store.get_course(course.id) + assert len(course.tabs) == (7 if expected_tabs_modified[i] else 2), f'Wrong tabs for course index {i}' diff --git a/cms/djangoapps/contentstore/migrations/0007_backfillcoursetabsconfig.py b/cms/djangoapps/contentstore/migrations/0007_backfillcoursetabsconfig.py new file mode 100644 index 0000000000..2123798d83 --- /dev/null +++ b/cms/djangoapps/contentstore/migrations/0007_backfillcoursetabsconfig.py @@ -0,0 +1,31 @@ +# Generated by Django 3.2.12 on 2022-03-18 13:49 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('contentstore', '0006_courseoutlineregenerate'), + ] + + operations = [ + migrations.CreateModel( + name='BackfillCourseTabsConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('change_date', models.DateTimeField(auto_now_add=True, verbose_name='Change date')), + ('enabled', models.BooleanField(default=False, verbose_name='Enabled')), + ('start_index', models.IntegerField(default=0, help_text='Index of first course to start backfilling (in an alphabetically sorted list of courses)')), + ('count', models.IntegerField(default=0, help_text='How many courses to backfill in this run (or zero for all courses)')), + ('changed_by', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL, verbose_name='Changed by')), + ], + options={ + 'verbose_name': 'Arguments for backfill_course_tabs', + 'verbose_name_plural': 'Arguments for backfill_course_tabs', + }, + ), + ] diff --git a/cms/djangoapps/contentstore/models.py b/cms/djangoapps/contentstore/models.py index f6817ac59f..66d884d8dd 100644 --- a/cms/djangoapps/contentstore/models.py +++ b/cms/djangoapps/contentstore/models.py @@ -4,7 +4,7 @@ Models for contentstore from config_models.models import ConfigurationModel -from django.db.models.fields import TextField +from django.db.models.fields import IntegerField, TextField class VideoUploadConfig(ConfigurationModel): @@ -22,3 +22,24 @@ class VideoUploadConfig(ConfigurationModel): def get_profile_whitelist(cls): """Get the list of profiles to include in the encoding download""" return [profile for profile in cls.current().profile_whitelist.split(",") if profile] + + +class BackfillCourseTabsConfig(ConfigurationModel): + """ + Manages configuration for a run of the backfill_course_tabs management command. + + .. no_pii: + """ + + class Meta: + verbose_name = 'Arguments for backfill_course_tabs' + verbose_name_plural = 'Arguments for backfill_course_tabs' + + start_index = IntegerField( + help_text='Index of first course to start backfilling (in an alphabetically sorted list of courses)', + default=0, + ) + count = IntegerField( + help_text='How many courses to backfill in this run (or zero for all courses)', + default=0, + ) From e8462bb6677648fa2b988db5ecf14c594b38d464 Mon Sep 17 00:00:00 2001 From: Chris Deery <3932645+cdeery@users.noreply.github.com> Date: Fri, 18 Mar 2022 12:09:52 -0400 Subject: [PATCH 08/52] feat: [AA-1207] remove redundant Tabs fields from courseware API (#30079) feat: [AA-1207] remove redundant Tabs fields from courseware API Remove redundant fields from courseware API. - number - verified_mode - original_User_is_staff - is_staff This is the backend work for https://github.com/openedx/frontend-app-learning/pull/873 --- docs/swagger.yaml | 4 ---- .../djangoapps/courseware_api/serializers.py | 5 ----- openedx/core/djangoapps/courseware_api/views.py | 17 +++-------------- 3 files changed, 3 insertions(+), 23 deletions(-) diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 148ac02010..48ca6e9ac6 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2527,7 +2527,6 @@ paths: as an object with the following fields: * uri: The location of the image * name: Name of the course - * number: Catalog number of the course * offer: An object detailing upgrade discount information * code: (str) Checkout code * expiration_date: (str) Expiration of offer, in ISO 8601 notation @@ -2535,7 +2534,6 @@ paths: * discounted_price: (str) Upgrade price with checkout code; includes currency symbol * percentage: (int) Amount of discount * upgrade_url: (str) Checkout URL - * org: Name of the organization that owns the course * related_programs: A list of objects that contains program data related to the given course including: * progress: An object containing program progress: * complete: (int) Number of complete courses in the program (a course is completed if the user has @@ -2556,8 +2554,6 @@ paths: * `"empty"`: no start date is specified * pacing: Course pacing. Possible values: instructor, self * user_timezone: User's chosen timezone setting (or null for browser default) - * is_staff: Whether the effective user has staff access to the course - * original_user_is_staff: Whether the original user has staff access to the course * can_view_legacy_courseware: Indicates whether the user is able to see the legacy courseware view * user_has_passing_grade: Whether or not the effective user's grade is equal to or above the courses minimum passing grade diff --git a/openedx/core/djangoapps/courseware_api/serializers.py b/openedx/core/djangoapps/courseware_api/serializers.py index 1b86a84bd5..5682e12e61 100644 --- a/openedx/core/djangoapps/courseware_api/serializers.py +++ b/openedx/core/djangoapps/courseware_api/serializers.py @@ -93,9 +93,7 @@ class CourseInfoSerializer(serializers.Serializer): # pylint: disable=abstract- license = serializers.CharField() media = _CourseApiMediaCollectionSerializer(source='*') name = serializers.CharField(source='display_name_with_default_escaped') - number = serializers.CharField(source='display_number_with_default') offer = serializers.DictField() - org = serializers.CharField(source='display_org_with_default') related_programs = CourseProgramSerializer(many=True) short_description = serializers.CharField() start = serializers.DateTimeField() @@ -103,12 +101,9 @@ class CourseInfoSerializer(serializers.Serializer): # pylint: disable=abstract- start_type = serializers.CharField() pacing = serializers.CharField() user_timezone = serializers.CharField() - verified_mode = serializers.DictField() show_calculator = serializers.BooleanField() - original_user_is_staff = serializers.BooleanField() can_view_legacy_courseware = serializers.BooleanField() can_access_proctored_exams = serializers.BooleanField() - is_staff = serializers.BooleanField() notes = serializers.DictField() marketing_url = serializers.CharField() celebrations = serializers.DictField() diff --git a/openedx/core/djangoapps/courseware_api/views.py b/openedx/core/djangoapps/courseware_api/views.py index f4bc772cae..9e8c1b44bc 100644 --- a/openedx/core/djangoapps/courseware_api/views.py +++ b/openedx/core/djangoapps/courseware_api/views.py @@ -67,13 +67,13 @@ from common.djangoapps.student.models import ( ) from .serializers import CourseInfoSerializer -from .utils import serialize_upgrade_info class CoursewareMeta: """ Encapsulates courseware and enrollment metadata. """ + def __init__(self, course_key, request, username=''): self.request = request self.overview = course_detail( @@ -82,17 +82,16 @@ class CoursewareMeta: course_key, ) - self.original_user_is_staff = has_access(self.request.user, 'staff', self.overview).has_access + original_user_is_staff = has_access(self.request.user, 'staff', self.overview).has_access self.original_user_is_global_staff = self.request.user.is_staff self.course_key = course_key self.course = get_course_by_id(self.course_key) self.course_masquerade, self.effective_user = setup_masquerade( self.request, course_key, - staff_access=self.original_user_is_staff, + staff_access=original_user_is_staff, ) self.request.user = self.effective_user - self.is_staff = has_access(self.effective_user, 'staff', self.overview).has_access self.enrollment_object = CourseEnrollment.get_enrollment(self.effective_user, self.course_key, select_related=['celebration', 'user__celebration']) self.can_view_legacy_courseware = courseware_legacy_is_visible( @@ -139,13 +138,6 @@ class CoursewareMeta: def license(self): return self.course.license - @property - def verified_mode(self): - """ - Return verified mode information, or None. - """ - return serialize_upgrade_info(self.effective_user, self.overview, self.enrollment_object) - @property def notes(self): """ @@ -427,7 +419,6 @@ class CoursewareInformation(RetrieveAPIView): as an object with the following fields: * uri: The location of the image * name: Name of the course - * number: Catalog number of the course * offer: An object detailing upgrade discount information * code: (str) Checkout code * expiration_date: (str) Expiration of offer, in ISO 8601 notation @@ -457,8 +448,6 @@ class CoursewareInformation(RetrieveAPIView): * pacing: Course pacing. Possible values: instructor, self * user_timezone: User's chosen timezone setting (or null for browser default) * can_load_course: Whether the user can view the course (AccessResponse object) - * is_staff: Whether the effective user has staff access to the course - * original_user_is_staff: Whether the original user has staff access to the course * can_view_legacy_courseware: Indicates whether the user is able to see the legacy courseware view * user_has_passing_grade: Whether or not the effective user's grade is equal to or above the courses minimum passing grade From 27769497717d2730e575832ba2652e6e5aec41a2 Mon Sep 17 00:00:00 2001 From: Long Lin Date: Fri, 18 Mar 2022 13:38:15 -0400 Subject: [PATCH 09/52] chore: bump edx-enterprise version --- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 799b062bba..27395ea310 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -31,7 +31,7 @@ django-storages<1.9 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==3.40.16 +edx-enterprise==3.41.0 # Newer versions need a more recent version of python-dateutil freezegun==0.3.12 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 7b8e73a345..e76c204248 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -453,7 +453,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.40.16 +edx-enterprise==3.41.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index ef560608d6..072cb32037 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -559,7 +559,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.40.16 +edx-enterprise==3.41.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index f2ae475e9b..2c203714c5 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -543,7 +543,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.40.16 +edx-enterprise==3.41.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt From 56cef9cb6a36e98ff92ea08ce139989f419c5ad6 Mon Sep 17 00:00:00 2001 From: Abdurrahman Asad <51022010+A-ASAD@users.noreply.github.com> Date: Mon, 21 Mar 2022 11:36:33 +0500 Subject: [PATCH 10/52] fix: add field to check if user is admin (#30089) fix: add field to check if user is admin --- lms/djangoapps/discussion/rest_api/api.py | 1 + lms/djangoapps/discussion/rest_api/tests/test_api.py | 1 + lms/djangoapps/discussion/rest_api/tests/test_views.py | 1 + 3 files changed, 3 insertions(+) diff --git a/lms/djangoapps/discussion/rest_api/api.py b/lms/djangoapps/discussion/rest_api/api.py index cbe509dac1..ac738f126a 100644 --- a/lms/djangoapps/discussion/rest_api/api.py +++ b/lms/djangoapps/discussion/rest_api/api.py @@ -308,6 +308,7 @@ def get_course(request, course_key): FORUM_ROLE_MODERATOR, FORUM_ROLE_COMMUNITY_TA, }), + "is_user_admin": request.user.is_staff, "provider": course_config.provider_type, "enable_in_context": course_config.enable_in_context, "group_at_subsection": course_config.plugin_configuration.get("group_at_subsection", False), diff --git a/lms/djangoapps/discussion/rest_api/tests/test_api.py b/lms/djangoapps/discussion/rest_api/tests/test_api.py index bdfcb6b318..cd8425bd28 100644 --- a/lms/djangoapps/discussion/rest_api/tests/test_api.py +++ b/lms/djangoapps/discussion/rest_api/tests/test_api.py @@ -200,6 +200,7 @@ class GetCourseTest(ForumsEnableMixin, UrlResetMixin, SharedModuleStoreTestCase) 'group_at_subsection': False, 'provider': 'legacy', 'user_is_privileged': False, + 'is_user_admin': False, 'user_roles': {'Student'}, 'learners_tab_enabled': False, 'reason_codes_enabled': False, diff --git a/lms/djangoapps/discussion/rest_api/tests/test_views.py b/lms/djangoapps/discussion/rest_api/tests/test_views.py index e7ec8a28ef..37e97daa8f 100644 --- a/lms/djangoapps/discussion/rest_api/tests/test_views.py +++ b/lms/djangoapps/discussion/rest_api/tests/test_views.py @@ -517,6 +517,7 @@ class CourseViewTest(DiscussionAPIViewTestMixin, ModuleStoreTestCase): "allow_anonymous": True, "allow_anonymous_to_peers": False, "user_is_privileged": False, + 'is_user_admin': False, "user_roles": ["Student"], 'learners_tab_enabled': False, "reason_codes_enabled": False, From 15ad3a36d86ebaf81f05fb966475f32d5eea7b01 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Mon, 21 Mar 2022 10:30:49 -0400 Subject: [PATCH 11/52] chore: upgrade xblock_lti_consumer library to version 3.4.5 (#30096) Co-authored-by: Simon Chen --- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/testing.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index e76c204248..40a064bb08 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -625,7 +625,7 @@ libsass==0.10.0 # ora2 loremipsum==1.0.5 # via ora2 -lti-consumer-xblock==3.4.4 +lti-consumer-xblock==3.4.5 # via -r requirements/edx/base.in lxml==4.5.0 # via diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 072cb32037..5fc4406aa4 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -840,7 +840,7 @@ loremipsum==1.0.5 # via # -r requirements/edx/testing.txt # ora2 -lti-consumer-xblock==3.4.4 +lti-consumer-xblock==3.4.5 # via -r requirements/edx/testing.txt lxml==4.5.0 # via diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 2c203714c5..db3cd65322 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -799,7 +799,7 @@ loremipsum==1.0.5 # via # -r requirements/edx/base.txt # ora2 -lti-consumer-xblock==3.4.4 +lti-consumer-xblock==3.4.5 # via -r requirements/edx/base.txt lxml==4.5.0 # via From d852e38cba2ed9655bf11b3a03f5685e47ac5e0f Mon Sep 17 00:00:00 2001 From: Alexander Sheehan Date: Mon, 21 Mar 2022 10:37:50 -0400 Subject: [PATCH 12/52] feat: new is_valid field for provider configs --- .../0007_samlproviderconfig_was_valid_at.py | 18 ++++++++++++++++++ common/djangoapps/third_party_auth/models.py | 8 ++++++++ 2 files changed, 26 insertions(+) create mode 100644 common/djangoapps/third_party_auth/migrations/0007_samlproviderconfig_was_valid_at.py diff --git a/common/djangoapps/third_party_auth/migrations/0007_samlproviderconfig_was_valid_at.py b/common/djangoapps/third_party_auth/migrations/0007_samlproviderconfig_was_valid_at.py new file mode 100644 index 0000000000..06bc446277 --- /dev/null +++ b/common/djangoapps/third_party_auth/migrations/0007_samlproviderconfig_was_valid_at.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-03-21 14:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('third_party_auth', '0006_auto_20220314_1551'), + ] + + operations = [ + migrations.AddField( + model_name='samlproviderconfig', + name='was_valid_at', + field=models.DateTimeField(blank=True, help_text='Timestamped field that indicates a user has successfully logged in using this configuration at least once.', null=True), + ), + ] diff --git a/common/djangoapps/third_party_auth/models.py b/common/djangoapps/third_party_auth/models.py index 1421767cb8..9686a756e2 100644 --- a/common/djangoapps/third_party_auth/models.py +++ b/common/djangoapps/third_party_auth/models.py @@ -700,6 +700,14 @@ class SAMLProviderConfig(ProviderConfig): blank=True, ) + was_valid_at = models.DateTimeField( + blank=True, + null=True, + help_text=( + "Timestamped field that indicates a user has successfully logged in using this configuration at least once." + ) + ) + def clean(self): """ Standardize and validate fields """ super().clean() From a23b9b748769ca276e6cd1e0afe1288941c5869b Mon Sep 17 00:00:00 2001 From: John Nagro Date: Mon, 21 Mar 2022 12:29:25 -0400 Subject: [PATCH 13/52] fix: release edx-enterprise 3.41.1 (#30098) ENT-5521 --- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 27395ea310..b2f30508cb 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -31,7 +31,7 @@ django-storages<1.9 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==3.41.0 +edx-enterprise==3.41.1 # Newer versions need a more recent version of python-dateutil freezegun==0.3.12 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 40a064bb08..1e1abe95c9 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -453,7 +453,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.0 +edx-enterprise==3.41.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 5fc4406aa4..0202b7d993 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -559,7 +559,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.0 +edx-enterprise==3.41.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index db3cd65322..c8a6f08aa3 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -543,7 +543,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.0 +edx-enterprise==3.41.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt From b3b79bf7540ef0ea62a55ab7f54c2c3385e4cb8f Mon Sep 17 00:00:00 2001 From: Alexander Sheehan Date: Mon, 21 Mar 2022 16:23:05 -0400 Subject: [PATCH 14/52] chore: enterprise version bump to 3.41.2 --- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index b2f30508cb..bcdd4d0425 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -31,7 +31,7 @@ django-storages<1.9 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==3.41.1 +edx-enterprise==3.41.2 # Newer versions need a more recent version of python-dateutil freezegun==0.3.12 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 1e1abe95c9..56dd90135c 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -453,7 +453,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.1 +edx-enterprise==3.41.2 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 0202b7d993..76d93b4f1a 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -559,7 +559,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.1 +edx-enterprise==3.41.2 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index c8a6f08aa3..72b91bbb49 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -543,7 +543,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.1 +edx-enterprise==3.41.2 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt From 5c8c69d9c272fffa5f0ff90b6db2013a073a6ec2 Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Tue, 22 Mar 2022 02:54:06 -0400 Subject: [PATCH 15/52] chore: Updating Python Requirements (#30104) --- requirements/edx/base.txt | 6 +++--- requirements/edx/development.txt | 12 ++++++------ requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 12 ++++++------ 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 56dd90135c..19b354f1ad 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -673,7 +673,7 @@ mock==4.0.3 # xblock-poll mongodbproxy @ git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a # via -r requirements/edx/github.in -mongoengine==0.24.0 +mongoengine==0.24.1 # via -r requirements/edx/base.in monotonic==1.6 # via @@ -716,7 +716,7 @@ openedx-events==0.8.1 # via -r requirements/edx/base.in openedx-filters==0.5.0 # via -r requirements/edx/base.in -ora2==4.0.4 +ora2==4.0.5 # via -r requirements/edx/base.in packaging==21.3 # via @@ -849,7 +849,7 @@ python3-saml==1.9.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in -pytz==2021.3 +pytz==2022.1 # via # -r requirements/edx/base.in # babel diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 76d93b4f1a..be64a3e5df 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -903,7 +903,7 @@ mock==4.0.3 # xblock-poll mongodbproxy @ git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a # via -r requirements/edx/testing.txt -mongoengine==0.24.0 +mongoengine==0.24.1 # via -r requirements/edx/testing.txt monotonic==1.6 # via @@ -955,7 +955,7 @@ openedx-events==0.8.1 # via -r requirements/edx/testing.txt openedx-filters==0.5.0 # via -r requirements/edx/testing.txt -ora2==4.0.4 +ora2==4.0.5 # via -r requirements/edx/testing.txt packaging==21.3 # via @@ -967,7 +967,7 @@ packaging==21.3 # redis # sphinx # tox -pact-python==1.5.1 +pact-python==1.5.2 # via -r requirements/edx/testing.txt pansi==2020.7.3 # via @@ -1133,7 +1133,7 @@ pysrt==1.1.2 # via # -r requirements/edx/testing.txt # edxval -pytest==7.1.0 +pytest==7.1.1 # via # -r requirements/edx/testing.txt # pylint-pytest @@ -1203,7 +1203,7 @@ python3-saml==1.9.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt -pytz==2021.3 +pytz==2022.1 # via # -r requirements/edx/testing.txt # babel @@ -1518,7 +1518,7 @@ vine==5.0.0 # amqp # celery # kombu -virtualenv==20.13.3 +virtualenv==20.13.4 # via # -r requirements/edx/testing.txt # tox diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 056f490660..8f3930580c 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -52,7 +52,7 @@ python-slugify==4.0.1 # via # -c requirements/edx/../constraints.txt # code-annotations -pytz==2021.3 +pytz==2022.1 # via babel pyyaml==6.0 # via code-annotations diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 72b91bbb49..f1edafc37e 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -855,7 +855,7 @@ mock==4.0.3 # xblock-poll mongodbproxy @ git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a # via -r requirements/edx/base.txt -mongoengine==0.24.0 +mongoengine==0.24.1 # via -r requirements/edx/base.txt monotonic==1.6 # via @@ -903,7 +903,7 @@ openedx-events==0.8.1 # via -r requirements/edx/base.txt openedx-filters==0.5.0 # via -r requirements/edx/base.txt -ora2==4.0.4 +ora2==4.0.5 # via -r requirements/edx/base.txt packaging==21.3 # via @@ -914,7 +914,7 @@ packaging==21.3 # pytest # redis # tox -pact-python==1.5.1 +pact-python==1.5.2 # via -r requirements/edx/testing.in pansi==2020.7.3 # via @@ -1063,7 +1063,7 @@ pysrt==1.1.2 # via # -r requirements/edx/base.txt # edxval -pytest==7.1.0 +pytest==7.1.1 # via # -r requirements/edx/testing.in # pylint-pytest @@ -1131,7 +1131,7 @@ python3-saml==1.9.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt -pytz==2021.3 +pytz==2022.1 # via # -r requirements/edx/base.txt # babel @@ -1403,7 +1403,7 @@ vine==5.0.0 # amqp # celery # kombu -virtualenv==20.13.3 +virtualenv==20.13.4 # via tox voluptuous==0.12.2 # via From 9b59b5e92a35811e7991449af03aa3cfd54e4196 Mon Sep 17 00:00:00 2001 From: Julia Eskew Date: Fri, 18 Mar 2022 13:11:40 -0400 Subject: [PATCH 16/52] feat: Add detailed logging messages about each course updated in Neo4j (coursegraph). TNL owns coursegraph and we've seen 7000+ courses be submitted for update weekly. While log message exist for each course not submitted, no log message currently exists for each submitted course. This commit adds logs for those submitted courses as well. --- .../commands/tests/test_dump_to_neo4j.py | 28 +++++++++++++++---- openedx/core/djangoapps/coursegraph/tasks.py | 26 +++++++++++++---- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py b/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py index 61349eda8a..0d03f56d5f 100644 --- a/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py +++ b/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py @@ -509,11 +509,29 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): @mock.patch('openedx.core.djangoapps.coursegraph.tasks.get_course_last_published') @mock.patch('openedx.core.djangoapps.coursegraph.tasks.get_command_last_run') @ddt.data( - (str(datetime(2016, 3, 30)), str(datetime(2016, 3, 31)), True), - (str(datetime(2016, 3, 31)), str(datetime(2016, 3, 30)), False), - (str(datetime(2016, 3, 31)), None, False), - (None, str(datetime(2016, 3, 30)), True), - (None, None, True), + ( + str(datetime(2016, 3, 30)), str(datetime(2016, 3, 31)), + (True, ( + 'course has been published since last neo4j update time - ' + 'update date 2016-03-30 00:00:00 < published date 2016-03-31 00:00:00' + )) + ), + ( + str(datetime(2016, 3, 31)), str(datetime(2016, 3, 30)), + (False, None) + ), + ( + str(datetime(2016, 3, 31)), None, + (False, None) + ), + ( + None, str(datetime(2016, 3, 30)), + (True, 'no record of the last neo4j update time for the course') + ), + ( + None, None, + (True, 'no record of the last neo4j update time for the course') + ), ) @ddt.unpack def test_should_dump_course( diff --git a/openedx/core/djangoapps/coursegraph/tasks.py b/openedx/core/djangoapps/coursegraph/tasks.py index f43384990f..bfca4f79b6 100644 --- a/openedx/core/djangoapps/coursegraph/tasks.py +++ b/openedx/core/djangoapps/coursegraph/tasks.py @@ -231,7 +231,9 @@ def should_dump_course(course_key, graph): course_key: a CourseKey object. graph: a py2neo Graph object. - Returns: bool of whether this course should be dumped to neo4j. + Returns: + - whether this course should be dumped to neo4j (bool) + - reason why course needs to be dumped (string, None if doesn't need to be dumped) """ last_this_command_was_run = get_command_last_run(course_key, graph) @@ -241,17 +243,27 @@ def should_dump_course(course_key, graph): # if we don't have a record of the last time this command was run, # we should serialize the course and dump it if last_this_command_was_run is None: - return True + return ( + True, + "no record of the last neo4j update time for the course" + ) # if we've serialized the course recently and we have no published # events, we will not dump it, and so we can skip serializing it # again here if last_this_command_was_run and course_last_published_date is None: - return False + return (False, None) # otherwise, serialize and dump the course if the command was run # before the course's last published event - return last_this_command_was_run < course_last_published_date + needs_update = last_this_command_was_run < course_last_published_date + update_reason = None + if needs_update: + update_reason = ( + f"course has been published since last neo4j update time - " + f"update date {last_this_command_was_run} < published date {course_last_published_date}" + ) + return (needs_update, update_reason) @shared_task @@ -366,11 +378,15 @@ class ModuleStoreSerializer: total_number_of_courses, ) - if not (override_cache or should_dump_course(course_key, graph)): + (needs_dump, reason) = should_dump_course(course_key, graph) + if not (override_cache or needs_dump): log.info("skipping submitting %s, since it hasn't changed", course_key) skipped_courses.append(str(course_key)) continue + if override_cache: + reason = "override_cache is True" + log.info("submitting %s, because %s", course_key, reason) dump_course_to_neo4j.apply_async( args=[str(course_key), credentials], ) From 813b403575a1c50fe2af657c69f4c7c885e55439 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Tue, 22 Mar 2022 15:33:35 +0000 Subject: [PATCH 17/52] fix: Use more accurate attr name for IP chain size (#30106) XFF is just part of the chain; record the length of the whole chain instead (which is always one larger). Also include junk in one of the test values for realism. --- openedx/core/lib/x_forwarded_for/middleware.py | 3 ++- .../core/lib/x_forwarded_for/tests/test_middleware.py | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/openedx/core/lib/x_forwarded_for/middleware.py b/openedx/core/lib/x_forwarded_for/middleware.py index 7d4447a8af..162a6e3d21 100644 --- a/openedx/core/lib/x_forwarded_for/middleware.py +++ b/openedx/core/lib/x_forwarded_for/middleware.py @@ -25,7 +25,8 @@ class XForwardedForMiddleware(MiddlewareMixin): # for monitoring in case of unexpected changes, etc. xff = request.META.get('HTTP_X_FORWARDED_FOR') xff_len = xff.count(',') + 1 if xff else 0 - set_custom_attribute('header.x-forwarded-for.count', xff_len) + # IP chain is XFF + REMOTE_ADDR + set_custom_attribute('ip_chain.count', xff_len + 1) for field, header in [("HTTP_X_FORWARDED_FOR", "REMOTE_ADDR"), ("HTTP_HOST", "SERVER_NAME"), ("HTTP_X_FORWARDED_PORT", "SERVER_PORT")]: diff --git a/openedx/core/lib/x_forwarded_for/tests/test_middleware.py b/openedx/core/lib/x_forwarded_for/tests/test_middleware.py index 52d1489eb6..8fceb2dc5a 100644 --- a/openedx/core/lib/x_forwarded_for/tests/test_middleware.py +++ b/openedx/core/lib/x_forwarded_for/tests/test_middleware.py @@ -60,9 +60,9 @@ class TestXForwardedForMiddleware(TestCase): @ddt.unpack @ddt.data( - (None, 0), - ('1.2.3.4', 1), - ('7.8.9.0, 1.2.3.4, 5.5.5.5', 3), + (None, 1), + ('1.2.3.4', 2), + ('XXXXXXXX, 1.2.3.4, 5.5.5.5', 4), ) @patch("openedx.core.lib.x_forwarded_for.middleware.set_custom_attribute") def test_xff_metrics(self, xff, expected_count, mock_set_custom_attribute): @@ -72,4 +72,6 @@ class TestXForwardedForMiddleware(TestCase): XForwardedForMiddleware().process_request(request) - mock_set_custom_attribute.assert_has_calls([call('header.x-forwarded-for.count', expected_count)]) + mock_set_custom_attribute.assert_has_calls([ + call('ip_chain.count', expected_count), + ]) From 700829bd4bd2f1b5ac22edef98c3abd035b72410 Mon Sep 17 00:00:00 2001 From: Thomas Tracy Date: Tue, 22 Mar 2022 13:47:08 -0400 Subject: [PATCH 18/52] feat: [Microba-1758] link new bulk email tool (#30099) Creates a link to the new communication mfe's bulk email tool in the instructor dashboard version. Staff can now use either experience. In the future, we plan on turning off the old experience, like analytics. --- .../instructor/views/instructor_dashboard.py | 4 ++++ lms/envs/common.py | 12 ++++++++++++ lms/envs/devstack.py | 1 + .../instructor_dashboard_2/send_email.html | 10 +++++++++- 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lms/djangoapps/instructor/views/instructor_dashboard.py b/lms/djangoapps/instructor/views/instructor_dashboard.py index 994320d350..d10c4f80b6 100644 --- a/lms/djangoapps/instructor/views/instructor_dashboard.py +++ b/lms/djangoapps/instructor/views/instructor_dashboard.py @@ -702,6 +702,10 @@ def _section_send_email(course, access): 'list_email_content', kwargs={'course_id': str(course_key)} ), } + if settings.FEATURES.get("ENABLE_NEW_BULK_EMAIL_EXPERIENCE", False) is not False: + section_data[ + "communications_mfe_url" + ] = f"{settings.COMMUNICATIONS_MICROFRONTEND_URL}/courses/{str(course_key)}/bulk_email" return section_data diff --git a/lms/envs/common.py b/lms/envs/common.py index 8b969b746b..b9acb1094c 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -960,6 +960,18 @@ FEATURES = { # .. toggle_target_removal_date: None # .. toggle_tickets: 'https://openedx.atlassian.net/browse/MST-1348' 'ENABLE_INTEGRITY_SIGNATURE': False, + + # .. toggle_name: FEATURES['ENABLE_NEW_BULK_EMAIL_EXPERIENCE'] + # .. toggle_implementation: DjangoSetting + # .. toggle_default: False + # .. toggle_description: When true, replaces the bulk email tool found on the + # instructor dashboard with a link to the new communications MFE version instead. + # Stting the tool to false will leave the old bulk email tool experience in place. + # .. toggle_use_cases: opt_in + # .. toggle_creation_date: 2022-03-21 + # .. toggle_target_removal_date: None + # .. toggle_tickets: 'https://openedx.atlassian.net/browse/MICROBA-1758' + 'ENABLE_NEW_BULK_EMAIL_EXPERIENCE': False, } # Specifies extra XBlock fields that should available when requested via the Course Blocks API diff --git a/lms/envs/devstack.py b/lms/envs/devstack.py index 933bda8e37..f30bf716d2 100644 --- a/lms/envs/devstack.py +++ b/lms/envs/devstack.py @@ -336,6 +336,7 @@ EDXNOTES_CLIENT_NAME = 'edx_notes_api-backend-service' ############## Settings for Microfrontends ######################### LEARNING_MICROFRONTEND_URL = 'http://localhost:2000' ACCOUNT_MICROFRONTEND_URL = 'http://localhost:1997' +COMMUNICATIONS_MICROFRONTEND_URL = 'http://localhost:1984' AUTHN_MICROFRONTEND_URL = 'http://localhost:1999' AUTHN_MICROFRONTEND_DOMAIN = 'localhost:1999' diff --git a/lms/templates/instructor/instructor_dashboard_2/send_email.html b/lms/templates/instructor/instructor_dashboard_2/send_email.html index b19d1ef4ed..51a67443bb 100644 --- a/lms/templates/instructor/instructor_dashboard_2/send_email.html +++ b/lms/templates/instructor/instructor_dashboard_2/send_email.html @@ -4,8 +4,16 @@ from django.utils.translation import ugettext as _ from openedx.core.djangolib.markup import HTML %> +
+ %if 'communications_mfe_url' in section_data: +

+ + ${_("To use the email tool, visit the")} ${_("new experience")}. + +

+ %else:
  • ${_("Send to:")}
    @@ -134,5 +142,5 @@ from openedx.core.djangolib.markup import HTML
%endif - + %endif From d05e5c639f1c58029a75de041a19d0ca47c9f501 Mon Sep 17 00:00:00 2001 From: Agrendalath Date: Sun, 25 Jul 2021 02:06:51 +0200 Subject: [PATCH 19/52] feat: allow marking Library Content Block as complete on view edx/edx-platform#24365 has changed the completion mode of these blocks. Before Koa, it was sufficient to view the block to get a completion checkmark. Since Koa, all children of the block must be completed. This adds a toggle to change the completion behavior back to the previous one so that the user experience can be consistent if needed. --- cms/envs/common.py | 13 +++++++++++++ .../xmodule/xmodule/library_content_module.py | 15 ++++++++++++++- lms/envs/common.py | 13 +++++++++++++ .../apidocs.cpython-38.pyc.139860693926064 | 0 .../completion_integration/test_services.py | 18 ++++++++++++++++++ 5 files changed, 58 insertions(+), 1 deletion(-) delete mode 100644 openedx/core/__pycache__/apidocs.cpython-38.pyc.139860693926064 diff --git a/cms/envs/common.py b/cms/envs/common.py index e4f050aa17..ebf11803cc 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -484,6 +484,19 @@ FEATURES = { # .. toggle_target_removal_date: None # .. toggle_tickets: 'https://openedx.atlassian.net/browse/MST-1348' 'ENABLE_INTEGRITY_SIGNATURE': False, + + # .. toggle_name: MARK_LIBRARY_CONTENT_BLOCK_COMPLETE_ON_VIEW + # .. toggle_implementation: DjangoSetting + # .. toggle_default: False + # .. toggle_description: If enabled, the Library Content Block is marked as complete when users view it. + # Otherwise (by default), all children of this block must be completed. + # .. toggle_use_cases: open_edx + # .. toggle_creation_date: 2022-03-22 + # .. toggle_target_removal_date: None + # .. toggle_tickets: https://github.com/edx/edx-platform/pull/28268 + # .. toggle_warnings: For consistency in user-experience, keep the value in sync with the setting of the same name + # in the LMS and CMS. + 'MARK_LIBRARY_CONTENT_BLOCK_COMPLETE_ON_VIEW': False, } # .. toggle_name: ENABLE_COPPA_COMPLIANCE diff --git a/common/lib/xmodule/xmodule/library_content_module.py b/common/lib/xmodule/xmodule/library_content_module.py index d3bce47bb8..e2b43387a8 100644 --- a/common/lib/xmodule/xmodule/library_content_module.py +++ b/common/lib/xmodule/xmodule/library_content_module.py @@ -10,6 +10,8 @@ from copy import copy from gettext import ngettext import bleach +from django.conf import settings +from django.utils.functional import classproperty from lazy import lazy from lxml import etree from lxml.etree import XMLSyntaxError @@ -116,7 +118,18 @@ class LibraryContentBlock( show_in_read_only_mode = True - completion_mode = XBlockCompletionMode.AGGREGATOR + # noinspection PyMethodParameters + @classproperty + def completion_mode(cls): # pylint: disable=no-self-argument + """ + Allow overriding the completion mode with a feature flag. + + This is a property, so it can be dynamically overridden in tests, as it is not evaluated at runtime. + """ + if settings.FEATURES.get('MARK_LIBRARY_CONTENT_BLOCK_COMPLETE_ON_VIEW', False): + return XBlockCompletionMode.COMPLETABLE + + return XBlockCompletionMode.AGGREGATOR display_name = String( display_name=_("Display Name"), diff --git a/lms/envs/common.py b/lms/envs/common.py index b9acb1094c..e8ebb068c6 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -972,6 +972,19 @@ FEATURES = { # .. toggle_target_removal_date: None # .. toggle_tickets: 'https://openedx.atlassian.net/browse/MICROBA-1758' 'ENABLE_NEW_BULK_EMAIL_EXPERIENCE': False, + + # .. toggle_name: MARK_LIBRARY_CONTENT_BLOCK_COMPLETE_ON_VIEW + # .. toggle_implementation: DjangoSetting + # .. toggle_default: False + # .. toggle_description: If enabled, the Library Content Block is marked as complete when users view it. + # Otherwise (by default), all children of this block must be completed. + # .. toggle_use_cases: open_edx + # .. toggle_creation_date: 2022-03-22 + # .. toggle_target_removal_date: None + # .. toggle_tickets: https://github.com/edx/edx-platform/pull/28268 + # .. toggle_warnings: For consistency in user-experience, keep the value in sync with the setting of the same name + # in the LMS and CMS. + 'MARK_LIBRARY_CONTENT_BLOCK_COMPLETE_ON_VIEW': False, } # Specifies extra XBlock fields that should available when requested via the Course Blocks API diff --git a/openedx/core/__pycache__/apidocs.cpython-38.pyc.139860693926064 b/openedx/core/__pycache__/apidocs.cpython-38.pyc.139860693926064 deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/openedx/tests/completion_integration/test_services.py b/openedx/tests/completion_integration/test_services.py index b24089c86e..0529ec7b71 100644 --- a/openedx/tests/completion_integration/test_services.py +++ b/openedx/tests/completion_integration/test_services.py @@ -7,6 +7,8 @@ import ddt from completion.models import BlockCompletion from completion.services import CompletionService from completion.test_utils import CompletionWaffleTestMixin +from django.conf import settings +from django.test import override_settings from opaque_keys.edx.keys import CourseKey from xmodule.library_tools import LibraryToolsService from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase @@ -181,6 +183,19 @@ class CompletionServiceTestCase(CompletionWaffleTestMixin, SharedModuleStoreTest assert self.completion_service.can_mark_block_complete_on_view(self.html) is True assert self.completion_service.can_mark_block_complete_on_view(self.problem) is False + @override_settings(FEATURES={**settings.FEATURES, 'MARK_LIBRARY_CONTENT_BLOCK_COMPLETE_ON_VIEW': True}) + def test_can_mark_library_content_complete_on_view(self): + library = LibraryFactory.create(modulestore=self.store) + lib_vertical = ItemFactory.create(parent=self.sequence, category='vertical', publish_item=False) + library_content_block = ItemFactory.create( + parent=lib_vertical, + category='library_content', + max_count=1, + source_library_id=str(library.location.library_key), + user_id=self.user.id, + ) + self.assertTrue(self.completion_service.can_mark_block_complete_on_view(library_content_block)) + def test_vertical_completion_with_library_content(self): library = LibraryFactory.create(modulestore=self.store) ItemFactory.create(parent=library, category='problem', publish_item=False, user_id=self.user.id) @@ -202,6 +217,9 @@ class CompletionServiceTestCase(CompletionWaffleTestMixin, SharedModuleStoreTest source_library_id=str(library.location.library_key), user_id=self.user.id, ) + # Library Content Block needs its children to be completed. + self.assertFalse(self.completion_service.can_mark_block_complete_on_view(library_content_block)) + library_content_block.refresh_children() lib_vertical = self.store.get_item(lib_vertical.location) self._bind_course_module(lib_vertical) From 0211ee6dcd614cbe4b9c07ede51d6f5d83942218 Mon Sep 17 00:00:00 2001 From: Thomas Tracy Date: Wed, 23 Mar 2022 15:30:00 -0400 Subject: [PATCH 20/52] fix: set bulk email link to open in new tab (#30110) --- lms/templates/instructor/instructor_dashboard_2/send_email.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/templates/instructor/instructor_dashboard_2/send_email.html b/lms/templates/instructor/instructor_dashboard_2/send_email.html index 51a67443bb..65726c9fe8 100644 --- a/lms/templates/instructor/instructor_dashboard_2/send_email.html +++ b/lms/templates/instructor/instructor_dashboard_2/send_email.html @@ -10,7 +10,7 @@ from openedx.core.djangolib.markup import HTML %if 'communications_mfe_url' in section_data:

- ${_("To use the email tool, visit the")} ${_("new experience")}. + ${_("To use the email tool, visit the")} ${_("new experience")}.

%else: From 4ed395a653d5bda6ddff6a12ae214571604d095e Mon Sep 17 00:00:00 2001 From: Binod Pant Date: Wed, 23 Mar 2022 15:50:47 -0400 Subject: [PATCH 21/52] fix: make completed_timestamp optional (#30111) in learner audit record model ENT-5622 --- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index bcdd4d0425..d0301dd8bc 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -31,7 +31,7 @@ django-storages<1.9 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==3.41.2 +edx-enterprise==3.41.3 # Newer versions need a more recent version of python-dateutil freezegun==0.3.12 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 19b354f1ad..1a1eafe3a1 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -453,7 +453,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.2 +edx-enterprise==3.41.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index be64a3e5df..ca40f56961 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -559,7 +559,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.2 +edx-enterprise==3.41.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index f1edafc37e..daddddfee0 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -543,7 +543,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.2 +edx-enterprise==3.41.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt From 2280420785459634377f475fd6d9ea929babcc42 Mon Sep 17 00:00:00 2001 From: Keith Grootboom Date: Thu, 27 Jan 2022 11:39:28 +0200 Subject: [PATCH 22/52] fix: problem_grade_report task parent dir being discarded. When running the problem_grade_report task the upload_parent_dir argument should be used to place the task output in the specified directory. --- .../instructor_task/tasks_helper/grades.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/lms/djangoapps/instructor_task/tasks_helper/grades.py b/lms/djangoapps/instructor_task/tasks_helper/grades.py index 166baf249e..0978d70a28 100644 --- a/lms/djangoapps/instructor_task/tasks_helper/grades.py +++ b/lms/djangoapps/instructor_task/tasks_helper/grades.py @@ -184,9 +184,22 @@ class GradeReportBase: Creates and uploads a CSV for the given headers and rows. """ date = datetime.now(UTC) - upload_csv_to_report_store(success_rows, context.upload_filename, context.course_id, date) + upload_csv_to_report_store( + success_rows, + context.upload_filename, + context.course_id, + date, + parent_dir=context.upload_parent_dir + ) + if len(error_rows) > 1: - upload_csv_to_report_store(error_rows, context.upload_filename + '_err', context.course_id, date) + upload_csv_to_report_store( + error_rows, + context.upload_filename + '_err', + context.course_id, + date, + parent_dir=context.upload_parent_dir + ) def log_additional_info_for_testing(self, context, message): """ @@ -317,7 +330,7 @@ class _ProblemGradeReportContext: self.report_for_verified_only = problem_grade_report_verified_only(self.course_id) self.task_progress = TaskProgress(self.action_name, total=None, start_time=time()) self.upload_filename = _task_input.get('filename', 'problem_grade_report') - self.upload_dir = _task_input.get('upload_parent_dir', '') + self.upload_parent_dir = _task_input.get('upload_parent_dir', '') @lazy def course(self): From 1cebd3ed7bc1646792ec273532135cf0540ac14b Mon Sep 17 00:00:00 2001 From: Usama Sadiq Date: Thu, 24 Mar 2022 17:21:56 +0500 Subject: [PATCH 23/52] Remove django-ratelimit-backend (#30054) * fix: remove the usage of django-ratelimit-backend Co-authored-by: Awais Qureshi --- cms/envs/common.py | 7 +---- cms/envs/test.py | 5 +++- cms/urls.py | 2 +- .../djangoapps/util/request_rate_limiter.py | 27 ------------------- lms/envs/common.py | 7 +---- lms/urls.py | 2 +- .../oauth_dispatch/dot_overrides/backends.py | 17 ------------ 7 files changed, 8 insertions(+), 59 deletions(-) diff --git a/cms/envs/common.py b/cms/envs/common.py index e4f050aa17..fe1dbb0a08 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -663,7 +663,7 @@ AUTHENTICATION_BACKENDS = [ 'auth_backends.backends.EdXOAuth2', 'rules.permissions.ObjectPermissionBackend', 'openedx.core.djangoapps.content_libraries.auth.LtiAuthenticationBackend', - 'openedx.core.djangoapps.oauth_dispatch.dot_overrides.backends.EdxRateLimitedAllowAllUsersModelBackend', + 'django.contrib.auth.backends.AllowAllUsersModelBackend', 'bridgekeeper.backends.RulePermissionBackend', ] @@ -810,9 +810,6 @@ MIDDLEWARE = [ 'codejail.django_integration.ConfigureCodeJailMiddleware', - # catches any uncaught RateLimitExceptions and returns a 403 instead of a 500 - 'ratelimitbackend.middleware.RateLimitMiddleware', - # for expiring inactive sessions 'openedx.core.djangoapps.session_inactivity_timeout.middleware.SessionInactivityTimeout', @@ -1688,8 +1685,6 @@ INSTALLED_APPS = [ # Learning Sequence Navigation 'openedx.core.djangoapps.content.learning_sequences.apps.LearningSequencesConfig', - 'ratelimitbackend', - # Database-backed Organizations App (http://github.com/edx/edx-organizations) 'organizations', diff --git a/cms/envs/test.py b/cms/envs/test.py index aaf7feb969..e58b8ea20b 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -283,7 +283,10 @@ FEATURES['ENABLE_TEAMS'] = True SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' ######### custom courses ######### -INSTALLED_APPS.append('openedx.core.djangoapps.ccxcon.apps.CCXConnectorConfig') +INSTALLED_APPS += [ + 'openedx.core.djangoapps.ccxcon.apps.CCXConnectorConfig', + 'common.djangoapps.third_party_auth.apps.ThirdPartyAuthConfig', +] FEATURES['CUSTOM_COURSES_EDX'] = True ########################## VIDEO IMAGE STORAGE ############################ diff --git a/cms/urls.py b/cms/urls.py index 3facddf0bf..541e2aee85 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -10,7 +10,7 @@ from django.urls import path, re_path from django.utils.translation import gettext_lazy as _ from auth_backends.urls import oauth2_urlpatterns from edx_api_doc_tools import make_docs_urls -from ratelimitbackend import admin +from django.contrib import admin import openedx.core.djangoapps.common_views.xblock import openedx.core.djangoapps.debug.views diff --git a/common/djangoapps/util/request_rate_limiter.py b/common/djangoapps/util/request_rate_limiter.py index cc2ad663e0..a9e9b6312e 100644 --- a/common/djangoapps/util/request_rate_limiter.py +++ b/common/djangoapps/util/request_rate_limiter.py @@ -2,30 +2,3 @@ A utility class which wraps the RateLimitMixin 3rd party class to do bad request counting which can be used for rate limiting """ - -from ratelimitbackend.backends import RateLimitMixin - - -class RequestRateLimiter(RateLimitMixin): - """ - Use the 3rd party RateLimitMixin to help do rate limiting. - """ - def is_rate_limit_exceeded(self, request): - """ - Returns if the client has been rated limited - """ - counts = self.get_counters(request) - return sum(counts.values()) >= self.requests - - def tick_request_counter(self, request): - """ - Ticks any counters used to compute when rate limt has been reached - """ - self.cache_incr(self.get_cache_key(request)) - - -class BadRequestRateLimiter(RequestRateLimiter): - """ - Default rate limit is 30 requests for every 5 minutes. - """ - pass # lint-amnesty, pylint: disable=unnecessary-pass diff --git a/lms/envs/common.py b/lms/envs/common.py index b9acb1094c..876dde9d0c 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1273,7 +1273,7 @@ DEFAULT_TEMPLATE_ENGINE_DIRS = DEFAULT_TEMPLATE_ENGINE['DIRS'][:] AUTHENTICATION_BACKENDS = [ 'rules.permissions.ObjectPermissionBackend', - 'openedx.core.djangoapps.oauth_dispatch.dot_overrides.backends.EdxRateLimitedAllowAllUsersModelBackend', + 'django.contrib.auth.backends.AllowAllUsersModelBackend', 'bridgekeeper.backends.RulePermissionBackend', ] @@ -2145,9 +2145,6 @@ MIDDLEWARE = [ 'lms.djangoapps.discussion.django_comment_client.utils.ViewNameMiddleware', 'codejail.django_integration.ConfigureCodeJailMiddleware', - # catches any uncaught RateLimitExceptions and returns a 403 instead of a 500 - 'ratelimitbackend.middleware.RateLimitMiddleware', - # for expiring inactive sessions 'openedx.core.djangoapps.session_inactivity_timeout.middleware.SessionInactivityTimeout', @@ -3198,8 +3195,6 @@ INSTALLED_APPS = [ # Learning Sequence Navigation 'openedx.core.djangoapps.content.learning_sequences.apps.LearningSequencesConfig', - 'ratelimitbackend', - # Database-backed Organizations App (http://github.com/edx/edx-organizations) 'organizations', diff --git a/lms/urls.py b/lms/urls.py index 0f249a379b..c1362779df 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -12,7 +12,7 @@ from django.utils.translation import gettext_lazy as _ from django.views.generic.base import RedirectView from edx_api_doc_tools import make_docs_urls from edx_django_utils.plugins import get_plugin_url_patterns -from ratelimitbackend import admin +from django.contrib import admin from lms.djangoapps.branding import views as branding_views from lms.djangoapps.debug import views as debug_views diff --git a/openedx/core/djangoapps/oauth_dispatch/dot_overrides/backends.py b/openedx/core/djangoapps/oauth_dispatch/dot_overrides/backends.py index 96f0141d65..d115a3a5ec 100644 --- a/openedx/core/djangoapps/oauth_dispatch/dot_overrides/backends.py +++ b/openedx/core/djangoapps/oauth_dispatch/dot_overrides/backends.py @@ -1,20 +1,3 @@ """ Custom authentication backends. """ - - -from django.contrib.auth.backends import AllowAllUsersModelBackend as UserModelBackend -from ratelimitbackend.backends import RateLimitMixin - - -class EdxRateLimitedAllowAllUsersModelBackend(RateLimitMixin, UserModelBackend): - """ - Authentication backend needed to incorporate rate limiting of login attempts - but also - enabling users with is_active of False in the Django auth_user model to still authenticate. - This is necessary for mobile users using 3rd party auth who have not activated their accounts, - Inactive users who use 1st party auth (username/password auth) will still fail login attempts, - just at a higher layer, in the login_user view. - - See: https://openedx.atlassian.net/browse/TNL-4516 - """ - pass # lint-amnesty, pylint: disable=unnecessary-pass From 856731fd48e97db23d5a8f21f1bd4a3380ab681b Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Thu, 24 Mar 2022 18:26:43 +0500 Subject: [PATCH 24/52] docs: Removing bokchoy and a11y reference from documentation. (#30105) * docs: Removing bokchoy and a11y reference from documentation. --- docs/guides/testing/test_pyramid.png | Bin 17582 -> 19816 bytes docs/guides/testing/testing.rst | 253 +-------------------------- 2 files changed, 5 insertions(+), 248 deletions(-) diff --git a/docs/guides/testing/test_pyramid.png b/docs/guides/testing/test_pyramid.png index f5d359bcfd6a8ab14025a3ab1c05bd8a3cd6738b..6100353fb4ca98100f9384de422b536100e7a2f2 100644 GIT binary patch literal 19816 zcmY(r1yoe~+cr!|H_~0wE!`m9(hXA5A`IQ#9U@3dcf-&k-Q6A1DP8ZLbDr~G-?x^t z81de}y!*Pg5vs~EXvjp!P*6~4a*R4{A($`lWNFp!Gc2#7940e9O#ZuJn_& zmd>kt8t1e+p+8<@K3QI4!ciYxXvAO|DJCXn9%!EyC3KMGU}$pQb5cNKSBnII3ma|H z3Vo5}T2UKJmJh8vYdU6<)sa#fBUg@QE9BY*-akngm!Be|0YrvCYDwV z_Glr`s;@%XWX&wJRiBqk@vF6zHgo0QpB}5oetdo0D?#LCe}XO({MC^rx&#__(jA9! z2p&Ir)w!UN(^hD?iCfkh8!HmE@;GHiB%pHGvGY2bQYVf@+t|J(!IaH$T~^OusjQTdF7z@?8)I`jT8h8X?8fNmr$YTj72Cz< zp$J2RDGKwx`oe5HS*}aP1hm~@uJR{9CUHgYw6GVKtg3&vlXpg3K2d=Zcmn-5w+6u) z9WaRqFv&ibB{B^=O#tijiI=&<(cQ2u7@3fgkUORbMzZ~U~Q7~_b86a$GAB2B!fAmz^>@A7RaAIqU3X`5Ga-?{%9pDM+SXwixfE>h3(DE9pc9G|Mmcf)pCnt-K ziGr7Llv^KGWMQo-M{|Pn>3}K1vBIB{NI516u#jlT0!A~?p8Z?5MbavVcn0&Hbs4gX zEwG)a(SC-!PjH>VHv0|Sun*#Sz)?^zzAS|&oDseKGGmRlpNCOP8J{4nClYBSp*01| zPY?Q&;-Wz~S${Zm?3@4#C5iw7h?aHAcEn|QJk(3rAAZB)kk@4R3HtssqMlN61lv-A zX5$NRx9C++BffTsJ*-XHF+D`TPiAG3)VjgJcgcFLCN1dhZ>kj?*odTqZ>Hp|1KOmT3Huz1pOC=|_v`8RkXf`wgLj!0PJ)l+LTju7< zbgJ_+0)~W3hB9w|y3LK!N3Mq#QlR8HSSG;hQr0ne#c%G%z{#3?1{Na8m4|_i7}-_2 zM;E#Bz0p}CRJ+y1w*_^nq0L7VScrt>K&%j4imoRWG&VsZ6xwlo_XolO&?V@kmS8fYdQz`1-qQ04K$WJk3R)?Uv&T567o zVie}LzR9?UNv9ON^lzgdOcLHum&dMVg??<5Qtxa-F&6;~aDI5NK5i&>2 zL1K4dUgW7U8E*c3fxlky?diAeftRbD2j4>VUl|yhi8LtvYM=CTWF$DHoZJGRhUj00 zewROCxa(+`_P=OdhYL}#4J4)WmLbcx$acZQeKxUIm3=$;9q#b^QfUDb=;&_^H8@p; z&|Zl$j1u=UP>9mek}M1hAt)XSiCzkJSWgV5heFlONM}VV7hT zKMB~7Xh;CX2xj_dNv{WIuJ&~Oz3|6H>Cq$gvcJ!Yu#OxiA$sgN$L^mf+Lb-hK}zWNcrOXl$*E5VJOUH5PEFb?21UH}EXkqMcXrSKQC z-#%sJZ*TH8x4CWUPv*Nx7hmg!Vn*jc%K0q`h_1nrlreh-ZSW)gl`+x0s%xqB-KZ?r zi2HLHg6p8$-6`rA+Z@^!1g+zbw7^HTJwk(ongr+@dpC#97S4Y*n7%W4RYTzWo!3bQ z6+!>I2^5|Q43EiMJDmq==prDNf%GD{I>m9$U60ob8GH^s7so0nxGn%$6nN$?IJ++T zg)e}w~aH;9TN-=Zqv%~M(xzh}9TW*0Y}rz9*+H8gegp-(m>kQp#0fc)Ve zFh75`HhhhPxoDt0u(e6ep?;0_m??+&Z_jiUw=%(ZJD?H2)fej(9%*cJV3r+I~|z6fJI|MpqKiu^`UX6oMyfs6GBR1{T4t<5Lj?b zJG|#tjCmlsiFnQ}ycwBHh@2wkb1%1^3UWPSEoYtX$N&2KOYxKUOP#uK>E{o%FLzOA zT8f^Rv*p5qzJKG-$o%BGBH$JRL7z&v#(iPXSWe2g3k+R`b|A(#?DqqqhZ-u>1gX$O zuuX>`5)L~E2&Q;Q@S3~G4zgE%OiK!%0y(T`WZZD}^3 z09{0~LVH1S8Ep@>B}7b} z)rG7A;-!4*MB%j<6eeM^8W@)ZTBt%3ls*%*y#%FzngsuAB-J{d&S-j5IHVD;kuPpX zeLTZAmVoL z^i|sw4_J&G-KEvPvI!~y9EtuQ;C}U*?C5oak>0;=aEPZrI<$Oije~5V1PA_x33a4r z${I0C${6(R=kNzviZ#F^Z5zTUioSz8%ve%+|Nf+pAD6%HLT0a@M(N!pVZbh)|0dOo4`i&!tlrT)hw~8~|W{;cq)`qs0_|74@e6y&4=;x1k^htaX-R5!> zmcZa z`B-YBc~9ZMf1Az{XdnBGS)(TY6nJg3V!Yv#g|Z0XRFlu}=cK~NmeqjlK(7{kT-(R_ z2raKGEz?c0Mue4Zbn|_AnosiJ_x%3Im&sHTW~QS-*$7z*$c{I$&E1EA{K2LyCG{vJ zn>^cbgmt_o)BEi{MZn>Z0d0Pig|Xm?d@^B6yjUKf3rh9aAa`Fap2o4nn)jpID4gI9 zzKVG2@61Nz%GpN6x#g;s^ny`Q3Ky~zG{Eqbn|W4DIK{1X6~A|<$VbM{O95kSF^|(` zic(k%_Gh&Su9Zm7Jys2clZ1(o@Q3G7lb_nW=R0a_C#+Py{#+`f()r1$p#Nwy+_Y5# zDH(SN)gp}ZgYJ@)>yR8w5U?K7X!Ntiz^=W~P~Waqr)*|M<3@4mqP?HAixLjV_k_^N zghS{S;k6&qeO=RJXiQXK%75Q5^U9NoZG_a`oDCeTG&su}T2QOQo}N!A!>PK`OD6B; zayBI0C8q!mDktCs$i|L8{1g7Xt=qpWCQsT}A~lgTs4rgAF%5lL+?hnSh^##s$EZs1 zHw66lQo&DpEAoG=aEQH8nLK0GXXt03$gM|r7MyarL5^x6y?$NvNjjRm5=KpFlt?*{!EKZIheRm3h7@U;A;FAg>G#OO|KA9CKgo@Jd|}q6H%sOo zBC&W~Bqf6wIbeo>k#ADM)Y3nteAQ|Cx<_CyVfwCxFBodbb9aP_d05k}7@d#i&q(1a zAiUHLM@r@Aqy89Cdo^_J%=k5WJ9fZXwlZCbJOKEnP46`m5CrOW3+tpYF$A<`j)hp> zH5^Fdg>gWm3%>5Ydh_2Gdp{3O@W(?9k%6E<(4b!dh~F$SAkIlQfM024$zCPqYb%(H zF#Qx%0n~VN7R$H-R@h>T`RF;UIm$T>d`WQ5_V~cqt_%zaLoHw@q^jMshiDT&K=(it9AM3m8|p zy=fcd>c#T;bK1O`W4>RtL|HOYBsJOV`6LWtDA7DyvhXGaWT+5d1HJe z2z8E-To{k}EnQT7Z4D^3uq1U*D&4S1iTK3)?T8m9oN~5Sv8uMJm8x4Hq68ZaCKn*i zAvqcxO7%~UHC!O7LsNEJo}s@t^}%*3q00nLRpwuL)iC(0aW5K71>lLApQH8gG+*ZI zy|`9>*l6n|GTCqE$-5p19-<#&9)85Y5TXp=!^**n!f}81Ncg`3 zM|Y;(WqM0qMJw`8U4{@<00m;sSk7q4n#8$;g6NzWT4mHR{+r(gMAa$?8l42P~z{B26)Q;d%dXX||g%T-O z-nfquMSv<|4mwcf)oU=ED9%)<{+eP#`tThHD?cMRJ93czzvWKSJ3KhjSTZ>ib{IM( zM%C$lOHjT;ZY!?m$l>_HJmiAjvD zXT@F^vkjXcde~_&@t_$StIs*kzW{ZWFVB4bQETl-;jCeNt>#sOEkwC35)e#9(LeronINRmThfxD`w>J1MU}9e8RDo zHgoCUoXXs>+|v+TO4uZ^aA28|aMEX+SdfJ5-BG`CtGRDW8ATFtZm>*JZWxs+J&G#i z0n&0W$&d{n>~Kh?I#6S-D2W*_wOExH;+D!w!Y!Sl`i(!>aqV5e0H&B>S(-o?I81;b zU55|_2^DCwxPF%Yivppn?1IFGF$6%g>iyxJlXWEI#A0EExJ?%#raXK)R+> z4bvduN~E`>9pz-$>GF%7yH%(knMw^FdtVX?{z&RMJUi)5H3hdWN0l;z|O*hA)_4n~lq81~t+7HVX|33v5h9q$nX=gfCmEW|Am3Cb+v%a`n3c?ikFcsp- zr_7Bd$3U|-7{b$67XW+FE9E<;9Vy*wBud{pA6>LqgfHBgND!mK)Ddt>{;rw2$|CvK zf%C|n$~@8|90fSsW|JnryoQ+8vgE>5eG$%v$4oPMK7)l9sUZ?LOupN1*rXtR)>YfN zYkw~xngA0dRStNC$d@Nb<&aS})9+O~Ep*SzT#`7M=J47sOVW0?6@~2Az8X!)rD0Rb z?078ZwjIM;y8d$0blCBDstzVwzTLWA^HLGM-$@czV`F6eh>MFWe%}Btio0EZL2wUaJLyr6PMLFn7M>`&zPK& zQefo1`DS)vBFD(*TC?HvBo$~CG|v#%-;$s7`Y_*FN|)s>;q!1%#K_Ojk4!80CE4%g zDtp$cpVTTP05Z zERXg&Yv(QhN#=FTk>Ak3UC#Fj*PUIrx4BO~n(V-Cy<_zbSmfIF(#~MgXsyTXs@wDY z^3qcDY?=1sI5p_J!qJE6qZ%3$+03=(=!s$RqpPPDgY6U2@5U;Im-_-ZQ_Og*p80`a zyu;(EF=b`Qd7}7!zr(Q#>o;9c+(4j`vZ@Zfll4x~=*<3i`>M5lWpCK1ZDw_?esI$S zm#7z&=EsS6%KGKHyZ@b(nH~EfD2asa)LDFF8%xH-lvJ|o4o{T@MXjg<#62Ya`h-y( zpsCDB$nn%5rRgCJD)0`BsXN-=XK#(841KRK)8X?tvMyEwZf%{VO*8o9Y6x$1=JT{< zTUFBeeBy}wg7B4n!t3@_69JusU?A#mreFEX^O;}9?YjFmTAg51(;sak52^O&o8@-4 z3~LNpCh#c2*LYG7G5kw0RszfC5W_~hbRurXX0@$9na;{_^dff~u-7A8<~t zlQDGP3^}9|y^j!heY(OOQUm8JiM~Gl5uM#lGp;nrxi}%Ta62f-R#NEYHED|<_yY^t zil8g4t>p}1pLDtYc#AQva`vmvL`eW7CKPXpXA99lR}SVtriX%J#jq{!`sy zipD#!Xq9d6`v-^mAWlLrJd}Xs>S3kDImXfNaXGYJ42@97%m}C8=r#(L!+q~s^yMN> zxN?}@LK3xJN-CCAu`s_Ai{i%>L-xbX|;ro1is!KXio#iMenfnGb zm;IfiwKd0dI+s;!jr}r@WOO4X`~vD#l=0KAw%an}=OLlA)Sn3n38tAYQ*tjZUna84 zKbfFe^Al@oYD&w?%VWIqB~IGgLy8*=?e4 z1F{y9p14o*y#qv3Ti4k#M>%GGMNS7Q+4pb9;j&_W&qIxv@&SPoEzBq|%n)qRG!k*( z`<`2F{P$=1Dul9u-uW0ZVLejk5spI4yS1=XYA_qecWxFIrCfauf+mkE%Z4wT5_K7b zhE)*f_qGta{q^P1vTOh!K`$hX+_#yC|Lac=BI+kHyl)tfe9|!ioXH`TA?Cu@L9umc4Wm_#{)z_hHUA7L}BuqGia(cGIjuPUE z*#zJZn!e4!ZdE*L)%7XYvhtbw&nKB5HF-V~)#80+FMQ@tW4goR>6kY?NAG|}-^=N` z_@N1Q8+G(KkGjuNNMqHU8eOK_7prY(YRXtg#Uk(Me|fp>Ji~FVMhS0nTbX)e3*X&U zB1Rq6Qu>A9^R9Dy&qHO{uMvF9V&iHlCLj97Z)_+*>kA9Tv7#^Tpdp5$Z~g4`LHx}1 z0cj|x#S%<->-L>D4OJy&m0(88s4vmO3hIAI)yrSc-Pl-v)qM^7h>!7(>x@4M=!1_j z--ZDZp#H21%Q5X6(zaHrQ%+Vt=Zy9=Bhp?zVH1NH+sxM2jgY z#aJVH`-VRBA}e0D>-cOFud^f2NZI5f8aII)&Cs^t!vfSUzL-R%WW%7fsl;}%dVlH4 zVP7|?BNcQlet7WAo`?3Eyd>v!$b7i2@~iL#ykptN17DdmzcXgbWEj(tO9CP#P#gM3OodDQc6B-|9 zFSYq|34g~09nZ->U2qkq1Ag;{ZV{w{cT=E~hWQ5s8B=j=0#J2$L(q!5*7?ZBVmH@Y zTf%Q-Nb&x?NaYT8<9f4GH{%VWi_RNAV(*v->4v6V@>J38eXV5i^Y|Mir6-6u#riNU z9Kty`&Hq+<7$(>7H@s1|+9A&5_mS3%45c@oW}`~#mJN+w)2Oi`l7!9*##55GQfRaZ&$Fz;LcC+`o{!p1pWpMmAK@p&=ZgUe zl)eNRGV8Vezt{&$@2|q#d;K4uuj$HM!ibvIJj)WYvK&JDXUG*fF8LM76U8<5lBv^= zYzLs|B%^iW{sI9es|)AzYd z-)((Q4%M%USRO!5JZk89Zdm13_KYBU7tjLLd%RdT+Iv#t4_kPd^Cn81`}6D#?^ix{ zv~>=>3eKu&MOe<GtfzyD9&wLS^-I&}~-9 zTX;y0(=!TdgI6aCSGK-aceR;qaoS@8D^7oSwZ1s9k9(<$u+t1_qZfCaAd6eb+5LtD zr6-rLC;+6>?FMNON|Ml(XB(fg4(wc=R&QCknkhp4C;2H;C@G2yC{6kB#h8<-0A`3d zgZz#YT{OY3C*4G0JJaf<4-m77A?hoiBD(6&IH_Y-- z<78RlLQ(GTF6{I)G$L^+fl3Jr#=7_O7}(x1v{n5?{(Ap_V08Z)u9LC__ewHdw(At` z9Cb_gH&=ue&Uo?a)rWh8cVe)D?8IK{9H=J{{)2}t>$0l)!I<2m9SXabRrFMne>>8~ ze@`5D_dC#f;UD-<)yZ*D5P7W^vx(lfUe08>Zmw|Jds7-Go)~^l7MVl@kn7}Q$}-`X zt=591EN>P!NGIl<9$7kh0nmu)(wl(9>gc18B1J|ZRyT87z~FJzB}2iT+hx&=(`RW4 z)gi^xTIS*dCuSse40M=hv{($4^^4+04~=*%>R0O|@8u|Rkun+o72d1Mm}mhR?R_@E zDIPiczm9!krba^}IM^aX%wo8lh80?<%-!fYwcXVBEDXMl9V|2i-$2nDvcqlTp(lB)x=L5i1fE>>X?Se) z4*;YbfT&2yCE=r%Wzk{`m^s-1Q`NFR5RB1~ttqgM03a=yRG>0jJ;BV|*ZT$NRR4gJ z?UII5j<@n~)opxl-f)B=vdx2?m3KoS2D=O!=A*2|bDCak+jmp?^!Jo|PfO<};YVU3 z6cVcP2GEhIccPvKUqo-~lCSWd>7@YkXewA@w%9ZYa}VtvdcGGN-R^J^Z&8?^%>Mu! zTs%FTc~_oZXZ1>m)ym1CC)~Jq-p;7pJp>1^u5wsOJ3bIPmrYa{?#(Edr>kI{#A_0nihT;W*!Xr@cO-#iX*(w;ea4J)6?f6BD`#aChuvp~jz_WOmHV z{9`8G@QegfOBDt*h}Trv+5IT6F17F$m%ha>!e0{m1hCX-Fs)%4xR6u!{aN>6_Jd{n z$M2|AfV%!tk($VC^@lyJL)wk1c%inTP^Th1s`KwJAE3E7v8$1wg&y-@&5uL0TMQn8 zCeV1WR%<>@b?G(lkN;=BH-Pza9AAI=T^$k?m6sqkgCvF^hW|5C+26rGHrz8$(PRP8 zudovb zup#E%I`tFA5EhfL&yA22?ofk`u}B|-DcQ8m=qqQ?M*avv&|;TNMR_q3K95n z{%|lz17<-c=pMzQfA&QM%IZA(!cjGVE#V7e;y97 zd?;Wha3=`U8OShP5M2hvd_9C4;fj0BhFgKlUU^nb%oT7Q@}5`AFn2MLu5C5;YXdG( z4CWcjf2{eFS^e57RU~5%-5Qb$~dyIMN-CIlay_8!(tz=D9VJ0Y1 z$}Y`KYV?feS8giAg|94UkiJkT&0t|d|-*Zi!%HUWELgPNXeTg*QAN6MeU@|@)`BSxQ}vlqo=oFYDxS3*@dJ!RZ(lp%tH605I8ZR zHWjC)lw$QOCn*ObFmT}=T}744nOAFm-7tIXTBE0D*)AN&U#w^T5ZxcAg1OShH?LS9 z^sJ<=Ctp7mZq=p{+C9xSsH|BcH>YRjY3`Xglq4$Pg>$E4^rh4D)fr*!>3kYvr}g+} zDpz6Kr!$g86;FT}n6@~@(yCsUbUjBCC*m+#{%Jn^t%?q|D&apYK)}@l$KXq_IHn?U z;<@=l1zfc5wiX>5JXYPy*?&(_!;IN8QTu(_P1fz)v{t7HfYn)2OSo77r3=FvP`E7} z#?xx9^4h=escU;cKM)jA78&`Zh72aatkzoBK;eD4oRl(U}&8}&fx$x=Y&ifw(xZ;uuk2V~YM=Au_RRHtx zH@)UNNr*DlvS?dWK0GDdUovP_@Pe>%e+L7Y7b-i%gU}js>C@rI?2Gv5nWBC)#|Fe{ zCX%`ircq`A`i#Rql{(#Z+h`p<0C2~^%yiNJ11jtelQI4aDxld4D4Vniy@LkAfhuD7 zr5gS?+cXMiJG=|Fi#q-00j~?7wYgj<0>A&m7Oaijmj1;SoDk6B$N;mVQtmo(DpAUd zN)+}KaKUlW_>E&4>wmF@U;`B@`5gilfX`QtlpLd|XV^AG}0Ms4tb$~q}+fQu}x3G8^@$Lf@#)t z<5R{F`B<^&=s*Zl^pgS<7)1aTj4Hs=SVCuM1CY^DZ6(#0xogFSE+5j(>XAayP5b&b z{@JT2%!DbE)!V_YPYons7T-h~=^;Qq*$m=4bcG&m0sMYutF(>%(Btqav@Ob{no1mVN2R<7>4~Ra;r!;6dKVXh=wesTn0)24U8smnp0Vq73~@y z4)01(P+xi#)j&Jn5U8?_r=O0)=V0)E_#fuL zBoi>YA=1v!sbD^5V*M&xr#8pIpmyEO@!h1P>!?Ect|bbO?AHI0wd&0w@SlM+fJ{OG z&q*SR{rSNzVhoRGi!Y8ZK(lOWl?2a&82k@ii@O#A{+;5g%OGF{8bRDRMGa!6NQJ66 zf&~6|0gjql{FIf6F7$wgmzz0x-O{Uoyz;jV4h|*9Y9lI`k`&wafEK~gK&>K>A{rw8 zKwSOizZm^F^|J{=cMG^iSb?|x|G+=oN*Z1#w7{q)$jN3xxlp)Jy5#Nm$>dVw2o0NB z#NnAid0&xnt*nACauntxd00zbtP0%W&j*Jgnz2CwGZ8Wo!Ky?IONwN8khv0;s!p^3 z?H=P((!QvA=nP3RBkR=R&wKwB7lN6HIND2T$)V0=`XNKnOfubd(^HaL0tzyy)aPhW z%@{i1d&btgv-Mk7211t$9;vXe2Q<#0kUk`fN!RMz zmnJ0if{Bm}k$(_P!r2iKqLsfjNP9UI<|K0X<++$uLXmX+s}Pj_3Brs|5Hr=P1)KZw zOXI2RPpx;*#pda}v+gHec-WgGPUcL_6mHciV;rbf>{iA6$Lxp$0HZDS-6J6E%wyp% z);~!N8W=kI466^xH>0$Tb=6afN|~Ig6;dINHQpZ4iC<1t3o{iLebW&@R zf8qtGgDi107=qd#)dqIvBmYQ8i|HC>>nwk#oLSr#LtGTgw}p5O;7Zi>i2E6EB+_EZ zrP!;7fw3TfYM<*&8rn5dxy?Kb2_gF~ZuN9tf2WSpv~pqiYF5Hh{y2{dYXg1Xu(vla zk7wHX(r7|9uU_kqSvC$1=!|2culM8f^JcHgce7Q#nyX)eyh}xQ=BkXV96IhT*R2hF z9ow(98tj)Fiwk`2Hp9vUPn)T2F1S8fVY+YkJRg?biq78;78$aE1=WjQ+?8e##s==Y z4FqgOBFlNF7N>kW?OHWlb(IQ~w~6#LUjdw>Smfo-?7FG*S;^3C6?FJV^wlF9Q972S zq~rDZtjsJuA*#Tn`km&A`JPsPGbNdF_rQ@ zEGc@}UT$%=(s7bvJO|)X8xqirTA_h@kutZD_eHO)l<={;jjio-X2*F58$svOE*shb zn8(QP8MIx|w08Mgpt4@r?64Mkx!dl;@I8m!2KtEcxMzyHR&Vj} z(DixXX#hTQj&rV8ZV=8`HM^udcYS@8H1zuFjr>{dpL=$sOy@Cv;bb>0tyId2ann9m z_5}b79ObJh-q_VYuD)#aygoDyA!eRPq*YKfbHAR|#uzg2MD$Enla%Z+nIzJ0wBvZT zvb01$93SG%?_M6dZ-}xJwf5#i>0S$dqlyvnA4}yzGrvcl; z8X6g8{e20()fMHv-Ed*xs18um-*O-M#z8quSs^()GxPRqb1sbcc{hU1Vy%VE)!wKn z7ltCr`eIf4jo$U74EYKMLz5EX*OWH^t&R8dw|jeB#UCh*Uf^zSD4`t8L7x# zjc@QpTw|-_{1?io`$hYz1FK%!Nndy5rNnJzhJL9e{%~R>h8VBD+L-K6@&vr=zXZV6 zsdmqyJ}JPXyGdB_CSFq&Hyf~OWB4TyY#v4ByR8Ug*A7N_<76Fmh$Met5#1XOgqr{h9sEq1 z0bpHH?t|(LsrQB!78cnDa+8W|nayV1f+Y6@jcm~S=KGfWpZ5K% zVy|s^4Yocsug>q)7c?$>TYR$IR2Ie7p#z{9RqGV?_V)79HKm;|*Hyx&5Lo%xDeL=p zn=zbIr^$D8--@1|uEv>m8X6ijO_2s8Bayj1F3q`Z=a@sJ7$uy(jXweyti15^-|5o2 zx@iM$6ecUBY>`!ipK$BJymkvYkx@~PABY*&HU}nT=#3B~pF1u zA>w3I8|VXrj!+2+cGI>u!^nAhOu-W|ylbC-78O07-R~&M$#HnUgv#%<{K?`n-+d=3 z&HK2WfSde)b`#_1-4h?e{q*8}aDD3qfZsR7GC$cu1(d9c|8mUVIA=d*rpxFe{*KoK zd+um=-?Edbgb}hvigXYj+)vlv9Qq1g4rQ}WNY8y z#!Na2wX@~n(y@{5?WU&UT^ETy51vWkeLW}N&Z)1j?{6VhJ_R4Kf?n?bFez%oJTBH* zWt7?GUWaLR5D^oPYBsYc-Uc9$skXYFaB6Q5(L2`eha~oXnBncbR|?ss^MPI2cm>c8 zwQrm=t-Is|+W+h-M@Wc_40>{QiJ07b>H3yebct2qsaSA4^Y6_0r%%PjSjDS1GsT)L zCsoGv&n2+rzF#SJn&{yXe~7i|RqHg!f^E?b@J^>CQo8y3wtAz`4q$jHo41f9QGZhA zo5n!Aa^FZg*E0)sNqDG>F#5NH^`|}FJ{8|f97$mO zQu`BZx+5>%Ur`x$g|_AV9fa{jop|`Wi{HsdK5&m$Pf%VYG1tzMn3y=i_h{P<-zQA6 zmXJ!RN=6nY4JAMfCKez88yu=gGzSyx)c=nx%gW-G6s@dq^-iNlh>}r{hKs}R#hX@v zkx^~KX0}vQIIr(^62ek>;<*Fx%GD>WNbVS`k#D(Vp8J%$*S?J{l76-fk5m}|aY5n; zK8k{BmA9)w@Avtxec#pCNGIO6oX=@ejJys~(RbEH4A!I9kYFrKq8V}BX; zQpVuW1`bfwqW+vNE*|qPDLl zF)KhEfxY`2{_+i%-P%O9*X1DH=4FNQ+b?SW4)&VDj)yl05$BGhN`-?cbUR9 z;(mM*gf!qw9ijEC%3A#frE^^5(Jq_9(9m$GDg2FI|6P_ye9v$UYAMnF%|G*+qGWeL z=l0|d=T2iBx_Vj7&H0pEE7>=E#3shq-Yq#C3bI{TYUQK%DTXtjagY`Fbd!_l#@H>d z7G725ls*zE?7q|cw$O?n)CBL`{^#XWohN}h;phoQs3sUf{}H!rb|2-Qr?=}2>sB-C zkG+1pA~jLjlf6);J-^Rq`ZD+TcUDvmIK2(qEeqI`sxYt`; z5Ks_#tdC|J*ssjcHGfQP4w-;wk$*o2O3lR*FT#k+Wi46oiZL5$B^e5_`5hKHu6$c_ zTQxrVLvTJb|4B7%Zj7_@=)qybi$&VU%*Vpp4K1@MS~xU+vz$Wm{bdTG z)7&`oN4Cyqr(5UO`)Vnd=MKQ?Ug!k(hEw4iPsbXu-O@biMSIFvvy2OmK3N}pQps@} zMqhQ<6fmBrgn##D8>Yw=ZrhK~H}8UqgM@zz3qDHrjs!k(u+41UTBp+-T= z&s5a@x#?-Ca@XxYMs=->N`Em|RoJxQw&sB+vCd+bpq-kxVPm`gcT<@W#84@U>Nio> zzSO(sH5ksp9DCp!-+%-u#<$@78mhP8m>L-E_nEln;nE%0v}4IMX-oCZzji@K2NU-b z&#uE66{yF+s}O4kr_IJ|4nsV1(ayWTs}M165`)Xjh#mKP8K!^Y;CbwVv0}Qk$-*_X zoFYH3xQi#X6d#WJz0?Uj?B_rJj#)!xPH+blo{3=fGau6cW{fwje5S{N5eP_#6^woj zk?Tug!HsW*}x-ldM+p}krT+ANL;0|$4*QG&PN|amGLYN}) z6M%;$wZ$doXjBUo{s0i;s!lr|9YU1gJq62GXi?<;c92Xu1Cvdn(yE$oUrx<<^uIlt zc&>(a9r|vMYUktpD0PYFyf>JHJTrp*XqRtMG=y1NPD@Mc6KJ9*!-qb6+HJb&4!*-d zG-!V9#W#MBpM)q?>gaQ%FF*H)#)#N>+w%@d$D#I?*Bv$RVVnGwKtpKLy*3bab0E9e z)GF;yaRH9Zrt`1#>Nl=~ML@m;p_(cYHjU7aaY9~V#BW4mqeHcGl)AgyjtfgUTYo-v zvve1dzsYUd|pseh8unFmw?QZsYyeR?l}`f}fp$@cEw zuztnl0h>OY`9+_q(>MrRnJpp$WJSX8T&h_K)+SRab#xEd_+SW7-AqC0TFEJ&;^^L_ zKtZAQLH;j*u6N%WDVuYgt}Lu!TmOBt_lziEOL+-`VKpb&RB_iMU#OE*Q>C5w{A21X$t{92d+le72ZJ-biZcP< zuwlVOPPn0-?R`wyuSYi9dC1ye?b}~GUD6-u3I=}L=tea_+GgSB{ep(9_A`aw<=v!f zS&ELO3zL~iP?E{^m>)>s`EzFjQpKHsL~8rA8WrfLbOHt#r-jc9qOGjL3Y} z%QV6SRoPLrJw^)y8vTd4mDO!C$=x$%2A0Y8?UzpNTUitTq_uU?1lM99ov2iAN2t#Q zF+9G0H|!c9*7EWMXdQ3CPBlOCj(f`vkB(A~Q@KpOemje--003c<~{0+36OS! z-G!27=&FnHS{rA*apD_iVZ1q!JZ2dgp`Qp5y9T^w}9=9 zXWURteLY&TBn7!030oig&fk|melJBbLIF-&I^Z+U6&diS5d=kqj5X@aa&Eh}i$3D# zhom%G$vGaLXBLCziTJd9-{;M(FYYpWq0k}QF~ykv_Wm*c74hH?_{`9{JNKoK`Zb`) zI8MHseQE&5pEKE^hfH1zzMNo+lRGD~z`|_6TL$*ALB6b;@2u?#Ro8g#%veu2RB;kX zj-Alykaz53(?~OrQKQ{!%{vOI2wx{}s+Wqqs-90J9?y%i4s9Vz`}%emD+eddC0_iU zR>2^E=QSTnq+G7JM*O3oj#tHdhCw9$jkn-&E zuj`MCzP2sG-1bgCYJ)@22ySypqKWU`bhHhGyGTmM5&yp)t~?&9u8)uPp^V2~jO~d~ zVHlc5lC`l6(ncDlk)`aJYO>VBScZ@-dwFDQ-{yQL{F=4Unaa+XHZ zdA8qf$lo}*R32cCk!zdY0pui8c+PwGQZxI2y5d&a(%PNxZqnnN_p$AzUiW99VvrG? z@6?gC(PsyNa7Oc=Na2^^30fW@`%VQS>-e^098p zY!a4bVjh_(rQ7+EufZ;UGj!TzMhFSl_V`Pcb*u3m6cJ1}N%DgehATmo z7lV_a>X|$zz}j$?k-i1J(A?t#Qk~!INfkb~x*GrN!Jt^$Bo@I#!IdE?2AAI)x_Z#? zrUubSDMlLS+$UFWeD6$ctKXv?!3VU+bgb8&AZ!}1`&H= zZ%FBEH;`r`91y4VVu;F7bwESQ)3$#HJ`hL`EExq7z^J8g`i^1I$sU<@g^py!QxeqC zTWxMM+?+|j>mC3xPaL24BP&%~G~<`;F~nP1D9^28905o&k58FtzBeu}2RSCUQ^z>(p7%sYW<` z|6GVYAxiXSf6MaKBK<5qxlY66cCuFM0mpP)PVO~z`1NkU2tAw&EqM_1QebR4ZhJ`( zuwK;Wf%2*{gkF6|Yl*jOXIhFmxr`|_)EYzc|;gK;J3T1=9hbQ(`x_!j2vOC(DF zw>-3MgR!#{HUcRV%m32Y@%d@nNS|pAbWOzefS_%ljK2tU+yCZ4t0~-=LNsP9IQQ`p z3fP4lY_q}-`IV^t&@%*1EGzS3q;Nc|0TG21U3edJd+W~txY=c+vZ6#A>ZhUJb|)&b z6PVaE7V4Xh>(q)L*&zz{z{b3(Q-OgPp90tnCOqZr((Rtd)(0GW*8TmTCp4|o;B$A& zZf0SPo~Ro!YeoKI?aaDD%FJVuaqsnyc|UtP=GfS_+Gk$OCk@S;$z$K{7azM2ykAWV z-B%b{IG^tlsyoHm_PcFC}#Ds`Z;V+@Mn zG~hVLItY-E$Mfzf$oakM(f7-7!Q~Ml)_t$lQ3?)3xqcUhID5o5O@S!Sl_y4%l-Ln; zpKaWu(`@dRq3Rl-_L}$Y;rDJc zo&?z>t|nC%J8>umIt9f+XCc<#K^FQF){9pcLmX>;!`~JnQV>Vibd-QY-k&K+pP_%I zFVQ6&yX_uJa}kcn$&csGOsn)&VoLmpil(~52KUH3t0DW;5FhSS?+rbWSC;gM<6_)>saZ)d1@A?QA-&Fh9hCnMrSxFGVOD-6q0#|{k$W2QK0=dCsy^nT%%}`bmTKi?u(TDzdd!aW=7>|gAFi+VFzVAZ zGHwFA5t_gCWckElPkcd)>%D>$Vhf(VQ=0ql+6M2%j-{SUnVP=yq^4*Y#3MPYEhw z*D`w5MjyusNB(wCRah{@?H5W{Mi{T0)R5YrI-813GZB6>$9_Q0&9REGl97g*Xkc~V z<~k|9ywmOns3=3n*e=5l5ikX8L9u`%xNG&*0NGUoUaCcL)9-nSi3w0_>}v2_s&vGQ z;zAp1mVeqyMby2Z)emjso%PwCg8y(Ij~69$5DdALB+Em0zy1=VG-O8aKd+^VxSYA} L@^or+peO$iYT=&6 literal 17582 zcma%C1y`F*(*}xrfa31%4#nNwHBg-5#ieL)_oBt!-QB&#-QC^3Jnt{~a!ztiviHu; z?#x`fd(9+VMM)Y35g!o(0s=)=MnVk&0&)rbW`>6X|B~;RGzI}7;U+5~s^I~7wg3kS zL5d7P`kMNa=_eWVn(V!cB5b|wHaRUS z7lsN-^H0Cczn=ZkBzhMT5S*>-dY@hvL5F5*fItU|gc$R|KANX9YyX46Lp6jA#x<$L zGsMiCZ+zC!j#w#qKG2(I!-F7$wK{sFEJYq;A=;b1_f2$iCZb~(>yQFBoXIG79>4Qh z)KQRqRE;g*&@6sh;fY6T6Oj%;xIi9w#Gf0a_bb`O3iuEZi{^-Lx|%?D@QTm;WZK}qyt`Kf;;hyf0)l%1STo8Lv;afIvo+}P&t zdSCO5JH6{x@iZD{5(qKm635M8_jZL;X*cGpx6qhtl+p!(Lv(Iha8I3Ii0zhNWaYj* z`@15KCnkpo9Zdw!pT>k*^6Tv zaQ_=uL%gxl>{Ol3eszhx+Op?dQRRrF)R!Uuv+oy9Z_Z7xvZbz_#fp!W z_!7>y^%xFtH}u@r$ggf;w@kzij$F>APYc-2PLYGCe)y3Pr!H>Z+$~1i*PTk@KY&8k)!o&n!Mq;Q7~|p( z`p=`-^h|fAZihOTsq3r>63k5aKBgjv*70{|3ecFqehr-6(3P!00<^HeABX` zfD8DqUo97^aPx!1Xi1^oa6vLU@#vm`qjAi$m!fanz61ogV?p1P(POB_iP^hj8ey!h zd8m)}$jd=Qcxo%y4^>PTk9_XC+WX8HwuIfSR)1H$FR?O0kq)3we#MrE1#=*n%`*rO%Jv+53m|*-GQk8X`1gySfu-LF;1P( zYO2h;p{>AAd^7&8J^5NUX*TZ%fFKK{UBzeSxQkOd@Vt{kT6A(Ja{2Ci;eA!oez;pLLiG&G0v-2o^4f~P<&xk$)PMMNhIS7lqY znMJ9}`=%*m7l!smn zKyainrwBamrv)JstjD)$8FmYdbMoOxM3O~`A>)R58pZkAY?95lN?GLdD$lJBI<6It zwVAk2B!UD8;ge*_;pVyv);jZ7 zzJWB}6YN$y+L>|q@#V<%CEStoN6dK~`|n$Z=P!+KhP%~g>o}@S1ug-~Feh7|EW^y0 zAaqO_mnD`mTpox`i+`>W@#NfQ0V`7hErpm}>sF~BrRU3O_a593BO|nBK3|Q=dQx6P zUz0^9&|C%1_B4vBj@L0EsDH4UQ@FbHiyf6p+cY(2cT z)$Lgd9H2l$Nbb9Z?EIwn|D+B0fcqotfvOq2A$NMCBO^9k-+n04S3$J6TANqtxY?Hn zk;`UcOs9Y4UM3B*-GFe#j20ANEgy}n;5qGN7`M%WD6x$WT#e)Yx z)5)Jxxa@XIalPVFF?f|wh=;#-YnS_-X`=qU6M|AJ3rOp)ofnE*lH-vfP%;S~9-8s8 z{XR_5jKa}@Rl)Vyy72R=Z2383!SiSUn`LB2AzJm{7&%`LOavX$fSct4XrHPXs*Usb_PefgE}fitfs7py6-r1*k)lQ$0OLJQ$mzA|hRS7F;| zUlgM(8=60gfVC)junmTD>P zII+FoiC0B^d%xEIL5xjWjHMTlUzr*>(d=5(d_9`%+iWaOzVQ)K2Ep*{;`KsBorrtC zO$U_vYo!52H<`6Kl>LnjqpA+%G+l@+{@zyI!SsA|;hMO)HD|LFey*N)=h`?@x*g zhDTlDPd^KrV*HOu`Jn(nC~H)2U)s0hsjwgJ%>rfWOcm97`e|+5)4fywcFiN$Le=(n97}As`MN@X%Sk4_cFf^anXaQF;<*Jr(jXOkK{PDvmc`HKb2&Rt= zI%GIHv-SE5qt?pIeY50Wq&^6RrvF}7ZBXesdH^rvc?^jxJoR72^$jx_QkpM2%H>d5 z8{2?Dk7Cfcw9M5?i0mKZ{8hNRIB3zv$UMlbpWFPqjV1cYh50r9KQ0BgGTOS}`~Vtj ze=5Bj-2S0j7OyIOHgz*=H?Fib8`?Bhu2(hD=kJ)es5(V@vwktXL@{;1s5|t2^iSyd zLW%qxhIdo>PTWYw=w^r~>~-p!be^vFgn$1r=}1v7d?2}$jnVw!Zq|>D+Ey-)c=`)x zB;DMz>SS-|IDg3zwBr0$Y27ce9UhJB(`6YlYJV)KU~c%lznP}xOHIn5W3dHdqbks+ z{n&L{(eGihcC7oR+HI?jhmq!Nmhdv1yCgS8{Y8y2%{db z@#ONdS67_8t=VxEl7$t;zI$OZKD!^6Z~nev*K92|fPNW&9se!Jz+>j?Ut8J`Ckkn7 zC#K8Qwe;VL^c3>73_znC;j7@yS>ddD<6Q+k`sZ}?_;ws%+=BJpY{|+qYmf4vF`o# zR{fBMIi@9tJgF*UYOZWx3(|MDBtd@3zNo>UCbKvgu)~*hhAUpoivhPJjE>7&@sI`f}q2^r2!-CsB zvQcT;Y~LhPyG2@E7nuVefh&q6?rU0e)!XjWuuaDT6t~THkh;%#)&5aNN^unoL)N{{ zg5GXV#>D^I4}DA}Be4iA|1cK27(8Ok&lRg*pX6k0dUZCf&tw%7##v zAD6m8-QRi{Fr4yaNRBs!*QO%g95Tz(?d)kDgQwlwB@1!#Bjv%aH$s9VmP&k27nd>1 z<-6O?%lqf_mZWlfBl)n@E!efG69bpPaktWRI$Ig&z?^yY(V>48cliPHjb>WA{w`;b z#r>%J-|FtcsJuIxlIKS(#r2Dd*VM>klR@6WFRkG>Pd3+U(;UKD48y2g(7g%o+va^q zHQLogjU9#m;&948B0b$WUAQO7%V^b$=goqDmHGhorBMpaOXFcK>=dsRw-YY`{@6lJ ziXKyu#}p!OL4O{_i?{jqf&7rd!oLBt>IqczHcF#|n}oMvb@|A8xT1hRA=S_YlQmB6 zkz?EhNZP7QB{w0-Vz2NKEuMiWN4Z%8=QB}K4@2?-sqMyH z&IBqio(GR?81+;+t1SX8R9g@*5xCYx*fFD6({=YX?U9qr1kNl^Lx+mZPTkl^QUENr zxwR?(bU7rBA9=aZt20~^j>f<5WwYBi%gtTk!O=Mh*{Jn+ehpzo*6Vn`K!CN^eHwp@ z=23hwnlX4GqB4LRXze!fP!6~~>zGHpYT!xD)fr~hX9;8Zc7nu}Yn=!OmT?t!b=bac z%3ZIkiwhY4%W(F5ssuBZwB+0%8$>=da%y9O-vBwdEpxjjkS;cu-!JrO6kDK(3oRLi za%3X0bJXwAP_umT=jcuhmvO^|qYdGL0^DZl;njtlAt9nj5|Q?p?Tn%nhG^d`#-5QB z37%B~gBEl1p@R@k{=(73=bh9Yg$y_G6^;VtvMxBJC>Y}|s}IFVZI^WB;n+Sqhgd5n zCy$oBzd`fOQSSi3!uiT(aLL7Kht=Bvd2=V+d~nsyd*d3>g5gl205e{nJJB~E0f+7Z zp9oIw27Ze|iqH04$Rut(Bnnz!!&QIDS-Q7il^IcxR6AUvGbMO!ezr-S;8wtaVFRCR zyHu=Y%VmnEb#DOtP4$?(B<`?J+1gwH0G3Ue3_EHA=h>biQHQ@noQyK>NGxH&g^%-} z;2N>QUxeX&uN{8w2=9k_dokqz58H)idu?)T;+tRL#6nI>&RCLBoUpi|hNpYYTj&mr ziJSDp)Xx2@Hqzb6sBbKf-uo`TFyQq8Hf>*Zt+LhIe2D_F~;d%(1Ueni@;?YH*Xwa^qtS@X`uFr z*fGN1lOtn$>si-{$OQpz<$06EzH?;~%r=w;y{^zOqdhbxdgpuk?vm32D~{{0YS7*W z5rxfYW)1>4lp(&Zc+{pz{34f{ppd{A{kxe3|8$VnEs@tdUzZL&*dtqA!1pS6zxV|o zA8ee?sijYYKU+{rF*>sG2Mv#0dIzycdB!Ar-y^V4o?*bT8vpJN%*7V6eZt zMmAp{cx!gv6J4{@pT2`y#?-+TO|m&hqk+o3E>U?XFx~^NTH@cW1o>Qu#J_PFR1JL3 z_Ch#fUT6Dr1vvH%pe&%@V2#9#ca&B3zE;cm4EuFH@ct&=fkB z_Ht=Ww<+7wH?iDJ0=%A-RLZd0hfT)eGYdxJuF%(@sSls#*PxLKdQ^I_fcQv_QeAiU ztJPh|Mnh(Z-anBc!A*GF#M`CG)D7V7&=LAz^yr7d7SxPuo4MXh?c2U&!j#1VYio+K zv=$X;6c>L?9zp7cjZ-b~6<$4fL61mMWnhFN2qX1fy?C|7d-y;Fx^$d^NxneGBMy!z zw|9_A6ntnHa`|FY)*F(fPlmUK0bSa@i-NJTrN0T7bB+SLr62>4eUR1Z3;r%g1u;7F#yji)_h%$2c#cU~-vvV(;S4L{h^noYf8ngFhfZh+g|c+NV3s znpU_v?FvjM~=MDiQ#mdh^DB1u&`+XY~50L&r*4xis&A zrkfZ54bK7OQsi?IE{~p~m8KD-2YI_QSY!>mEA9fJ+;A!bAX07q&8gJ~Eo2_GwJk_h zYxJhpAIK;&fC7c3FK6ooEjTtCyM4n&>L85rc5JLGyKX`sG zniUQiN!%YZI6}-KX8tR5K|=R%tcyJ09%0i}ubo!?#m&}iO=i8#jYd+({qiFUhK+Tt ziQ1U-z$fVh_vhigjEt51M6d0LqQ<~T%J_C9>NP&Et|BIX6ol{Po<{o;+n?I?rjyEM zztkC6@l?;p5Yx-e*s)?N5Dn%xr41;(_7V;w{DpGGL#4yce8iAGMWp*v&%9N{%UT zl><_D(&&fzb$558^!WXEID!?%%juiUgpJ2#{l%6eT)L3m=HLCWq zZC;pa_l5TUJTBdk`)xlfeF-C35FR_?u6_$Zzv->Mz?zXcRKhj6Z}mOaW`DM^3n9_r zjgBiGo{TR|nM@S8$A3KF2$~lDn}E)~Wn&W8Tud{)s;$QuzZSf; z8}@Lq=5jR0d^U|op<7z8AsF=AB(V8v)li*)qoE8Dm)U&sz*IGAD7BlL7r~7N$ z>7e`~qB*@@#M{73qV&V%o9(IT>o{Kq@ckI^!qBPn!fSo-nVXS80g>O?YBQd~66Tp+ zk2vdWY4&~b4`QT5#~3XTtt>3M^`h=jOhT6Iv!cZ%2JUmU^KwGcJ>AJ|5N*^Mv+B3- z^JTX{Y__21eTuhSHX*jf=CT~THQ&{_{)M2ja;!I6&{vIEZkFew$lWT#c2qv1?w_|t zVr@01u}cMn2*@gDU4Rf#Crkhv1|@C4PgBiSVJ6I9Vr)+ZB8Vy7i6LrzhWh5+-@yRLaK{*Yih=$8;$1HOH^{I4%`fK z9o{5}fA*_cOWpb3k$`(@u$t6xlirC*Me;els-96&+1E-0H}Lk&ub-Tjgpm9lXITb1 zduURY-8bqUqs1>MzlAFqX6HY}CsEK94ZZql}J#ye9Fj3C`Y-5!uqPuYp4hM`hX8-*IFsiIwwb8UCbyvlPpEt zYH?7;+mUsi_ALiQHmxnK#4%IpKJ1$H)va8@$cJ=rEn=8?M}jmwD`WBqEy#cuitO^Q z_v~_}$?t~otnY_tQaxD@IYg|%!4;G&2a2rB>F_`;dMx_Fjb>T*rhuCvACCZh-iOK! ziNw#n-sy<6UFf^JH$;TLm_9yVXKyjN^0Ac&=KvSYCo98)vXN244R61Xum+ znF3fd1xn}FoMc&c5yiM?zIiBXjgO#tYb@v1-|R0qFB2>H5);bIoNX-67uCUWLe+{4 zxn96al8YHcY}dnm)o;)7U)qBRWQ#xkID}GLN^SZi_Y~KyT;MA)6m@`F$q=j;YEgp) zUteO`%ZA;XKO#+!>TWfZe|_*7T8x$a65PzjHjhrEfnj$zCGqr?>N3dtq@+jD#O~lJ zaz`4^-j*w)S+vx!y_t(BKXAMgqbJa??8Ho21{p??OCfO}Bl*Vq31SWm_8m|IzntZ{ zzXsqg61f#iA$GoEift?zm@8R>FMF!Wm};o|>@%!*ghdvmzs_pABFeIvA*RTjA$gr`&wDa4Kab#D#vhSsWbapyVZ31=2-4`Z!wus&$B=@s_f{MuEV}6$W)&jcEdM zAm$>U8V4|NV#Qr;|H|GykkRC~5n&k5e9&Ww#Qs+tD4~HyF*d~*S5o5Xcw|JVAQUl* z&Lf8d-Ygrnny(em%~5bUJi-I5|s|l6U&~Lw~MKX87_WSY8@H2-54h9nHh&| zY|mTlp3U#O)I&UdN0L-l0K4VO>~9Bfp#|zSH&Yi(c4%fs>6^U%yEjCbq0{+%8AI{i z%Fh0~N+`C&uA>*Vq|LHZOGR#>xzm^vOX}5DvF1$6D)?97X07GD?%f7jL3E5xBR5iY zAex1+sht4nFJ+65f(!QbeKITn&O()9Wk&n1TdfupashqBX3;&*CJhHCE#1PGCFnNy zT!FNdEZe!15g_S(wWs;&k#DpX_*8v zl6-i}OeDnn2a9@Xp(JZ+wAZn`ToXwwX5h&ExTZqi#uDR6r35C6^~}H_QCtxw=#N+Xq}U z3J$)6eR}Yz@=KnPZI^~50RVi`wlO)eY+n~w523>5HUqQ=f|ycb|#ryjpGM*I(REK2A&9YfVKrd^Sf0D&|TXD+FJ&*xvP;ZAs{g z3H4M@mQ0m{vdlOaMy4LKX$szB)I}G1qA!g??(Mt3%FCO4_|&2t-$fW_q@4hiX#$eP zU9WRUqwGf8tI;E|ZO{8riem8Qs%@Js;R;K~krgY1PYqi^{)SwdEIUL--*?>y=^2)& z?LS^F;dMa_r1j(je*S0RO&jqR= z#zRQbL}l79A6f>6h6jg^!lRW{u^q@OY3=)Jexy{c8pdcj{`?28K7@8Ym!I7DMOd8W zPuA%wK}Fl>H5nKBX-gVJ=i#B+nEOi26-18OJ&t{)>1=Y1-R^iWVTSzO1WT@0osaT8 zX~FkXZFSd|`x1^fxI3%Ws$rL3%X+)n5kh63@2ySl3{PF=kYh)Fy1cR^W&W-S+kVkp z9&cytz^yc8`nAdMTR%%3eK8^C0?ym9*x-hbRkk?%tBoq0Yj>LUl5D6e{+gAI#1rRK zB_mg!Eh$Sett>`<6Q9_3J4uZFRD882@_`v4w`ZYW!9CAEce7)Dy)08CICj&R_=C4JqkubkyyJA5aP7yQw zNRm4Ji}&>sckrM6mPfzIKVFCwSs;Y$WfwXu;%{&FR=`~E?sQpp`)dL9k9wyrD!Tl| zE4+3sMOLBlA%RG!x+^kYew@nEc^4#Q7{m4U1l$_GkG~LfT%?=tlS#!}K@Glc%YoB+ zz0y8zJJAv-Tvvym-;|M&)85)q8GYuv;MnGz-b>MK@c&-$UB(0d@m>+*`8mE_6X_5Q zgAf>YD6vLbyX(~nW=aQN=MSec5NmxRRp+&7BK{5)!w|idFwp^*^e?Z)NKh|D0(gXe za~vj)rqoNk(ngfbWa`^yt*b{b&cCX?1$v^=u;I z7m_s6Gk%3&sQ!q66je)*=&!@qb#{)3Z@5Ed(${EW;u4BF9FL8VPsaIFoIF`rmR0o= z5`u*6l`Zwe4=scZB)W_EUF(FlIov-v`M%w+Z>+o^z0IPWPB*J&Dy8N;doq_)7?$Q% z78Q&ultV(CNUCX5szn{}NEU!+bV5FkSP%8~{6S{Q2gBaNFfo3x(WEgRO=)#IxAQDW zJUz`q%~t3kee^jluPnzFmy|KyQyzY!9)VtPI9g)ux(p>cJ0zo|#1bcjPf^$Q*5Kh! zV(kCf(^6npD5Y)iTRc4{F&P&V^X&5S6eo#vgx1DS^{paG;;Enuz($9-v=yb7mYR-B zK(JFyqZCK2emrsYw5`UPwT5XjXDAa92PS#yZz!RJmu-M&GS~rlUlX zlw^C)wK*eh4bavmr&?Mp-M3D8MRl4QMizN@ZDqGor*q#mtD`F$Dx^HyXVua?L%5@3 zmcbOmC}5KjK;4iBnvCjf2IdwiD@d8T2y+&SLSsu3}h?c8fOTtmZ6eN;sJp<<0eV~DjRW|-Ng2@3#c9j=m$qZc4 zQ5aUIV?7gm;y2nI*u-d+BE93S$MIJ5MlESTfo?X8r7yHf-Xt96ipl^AOap7c=~ubv zDn61xc&Phb(|l=aYSmoOY4t| zKEeqIg@6dO43;S}`nlkq=n^MZSsBX#cI!3N`DUi8s8J~-3V7*kiRVEh<*G3`qHKIx z;M+kMtSg88w-`@KAvHND(W^7~px~y?Y}6kJ4HqJST_-JC^?u24B5jHiI&|pi`0`)Y zE)SmTgWT?Jlt2kuO{TC?iK$-x(9$D#adg*fr&tM4WWQ}E15^Yq8cSu&^Ewq)23C(` zxEv}AI0w`}3I$VxO}H~?8BP~SUL}dT85<U zaN`@sO}VEG8GJTZ?iQEQ2biDgg^E}(7yz@m3C_Pgp5Asy)-slf9V zya+WDTiuPs?H>JV!*oabbRiKSR{2L6I*_#Dm-yqZ#j=-F0Z=wz7>lj$T46>G&hM9o z=B+rKXxzDWU$@W^Q>mY^Ar_n8uhsSFkfiMyb23Tvf)w^^p5Lejen~9q)^uuxV7_rD z!Jl43>sEe(=Rvp_K5$_uJ=w4%&EfFvoy&lICMQ_z>@5}xiZ#7Al3M(eI zKG>XCgi?3|6va$syLR!H=wCJ_`tOJV+8w)fBet>?t%~oQ2;0P zrmL(Pg*5%+?l3%1RGZe zd)YO+dIPQmXzc=QxBwneBun^xB(&?hbd2G5ZGq?XLlg&({?!$;mogR=%r!3BPT9dh zT!5|X1ULthIUUl2hF__>c@@gjp(;nTj+r}Y2vl;R6BNnt;YI%PDV&26YR+YaKBMzM z8))Kp?Y~HIc>V}RbTf%Hte=M;0&MjU9tB@b>hvhS^eUZf;)c?ZWmL2%)p$nygXs&Uj=)bU@iyUt=!j?fEgvjxiI}}f*_%R(5=&W3uOGV2vV_0n{Fpqg-nu_s-Pw!N3Ey?!F|cZvsmevaH;zN5RfLi%rXL}*P524DkY=l4 z)u!w6QrDv=pqMIA23cdk&n*noz9rt|J+~nV!-P>!SXK8Rk0i(V;Eo(h#h^o5uP_Vs zWA?Z*Z9WP9K!|AhRb5!+y!Q!)^{P4U98m;lQIj_8C@F473~=U+yEDf0qZJ!r)COT0 zBZ5?>{ukF(yZljLP={bR2E`8SD#}oPA*%*fDit?l9{gNI4V`X(SNwvk28bGPJ?^g>}nLliiCI{nWMr)*aLI^mc zb58QlL5x6ZK&%uI_SpW1L|zC0n(f0Ed35k%cKLU%>VTVkmjsM&EI6xG&dCoBLQ*NT z-w#SqIB?Yae?X^ZF5+FR6YFvGT>pWM$^d`p@=q$L6=<4&Zm3_S2D}{xdwK*Xc%;Fb z#tpn4QT>MoscTyzt&M<+IKYMAt|0Y88O=|O7_3oT#KCePNOTyT)J(!46F2!ZCk@RS zgZa(;zcG`wVCaK&iQ63{@SG!Um2q$ZaELdD7mv=)$|ZAs@PQ@TBxuQr`ySyNk)Omg;IYIG*Gl*@b8bzVvcc0B#p8S*&~HpP92sPs*650+{$sZU$bvEF zy+Dhpj3@Zrph`pjG-|{fhsCH1%D}l5J%PN)EtG!KTj9}&rCUv03K>sWP_raYk;v7_ z>lYzxB-&c^U|#s}l1-oi9rEc>Gk$&|@zVV~x;!wjisI40$#OiBcisJcPFX+kMrZo- z79=q^!*90A5NvjjXRK#Qd%N};ua`T&bv;VEbFdAZPWye#eZlO*5q%D7wP#6J<8bI+ z{5QVs;TH4CU{F4i{^G2y7V#_RV<-c^bE`>5Y2>+hHTA}QdNdwUZ}H4C)3L(+y1cy> z>wESvgj&YW9G~aY?~4MDmz(YHGw;=I9PrYSDK5=f?JkE8odRgp#{7-uD^_X|%;Lkv zXvSF$1uFX!~bs%Q3IVdKcp{8jh-` z*<*UelQVX`{Oux!)wbWP)EkJB!rSV-qnUL3b9sri7Xwgm%HTy}*8vUi{Z*N-IT#&jiI=GDzCE=hSH_+| z>W`V9AQ3!X?Tqt6gDER{lRrAz{)QR*pw;$GaO&6VNTYwL z6nJz5*Gw!T*hK$m{If}Y2oAyFQdmiY2C^T}#mddsEjSx81T%Rc-Q1I4ZaI!XmOE0( zj?V$|sYhIyyN>6L=?L<0O(x)vrL5U8oR~}4*q>YkfO;&I>eEetqydX2CYF56lL*`q zyDYehaM%KSs^*$^J6$6dTMCf5GFMZf7A>S@!(w~VNWx!5$%W&TvS=c&Q#l?I-(eR@^xL;-47tI`VynXMI6U*vw7fGfXb@rH~LWGxg&dHe6()neQZzwo(W z(FFnQk2WKdzCS%(vm{$st&-iw&fu$rfXWMKh4Se=L)Sy`_5h5P!f$@=5i0sC;YCTq zLpRk-Z&nbc0etdZP6x$m0ZbPNBPs-SEdvK**k^8b@{xXr}9F?67(hy zJB^k;iF9hH|9Y$)AQg14VkWtmt&0G)QsZ_)ICU%WESff_p5nQnAms|;h5B_0t*E(P zikolx-N%0e)AIM%3@`RIf1lRc?jcQMqGBG~M)+$@Tz*Dzm7@M}=LUiJx{u9A+x13$TQVT6Hf#Oj#C?g_ z7?J9^bK`_2{)i(jc7yP;vrU0Y%^6QCN#C$dz61AnZA?2$cP(N$a^O{cCgwG@rubb|Ejj@=qZ*;3cRNX)k}$~ z+1Tt)(@OJF%5)}Y=Gh_eyI)i@8vuMIro3kc?&70i)5HOgA|(Y3THMuL8^2viF!arS z__YQ-E{0=EPXCyF9+^zqv2@9i;dap2WG83773Ih|S*!AhYH9W_G#l_&u-5I!NByat zI94gsx^-_EDewt7e#Vne_@J1 zLGE@gNg4AtnJV@R|LmEpxGdW@*cjR5%zxr;7XLi?s;k*;^M8>8wH(cO9+PifAE|cg z;EF#iEBE?kSrUL7S^g<`g(KUI_VXFIM@y&6GI&h>jcPmFBFM;_TXH0#fYn0vRW)5q zXFWixAJK|F=9lyKd(^}Xo%n@Uk6G0BZNy4MN<-S5`e~#H9G;|jnRk6&9TfR zOIK&VV+H!OB{2c4?~+mND?q=PJF^wz(kBSw+lhsp9u^Q57|k43Qlg9pF#5Y`o( z+RjBOks9CXH4P;f`Z}(nZiV5XuiIpmL4I%|xb_k581$F@bc%tgQ-hG=(CGShZ5aZwDBN?`a$% z&Jk&aAR^%3ue{`Sg~{I&vDk*M`8s9UraPODATN8fZq_;v4{qS-@=g(A&E<5b{BSH{ zD5#Dem+I9mRt$V-XB`hkqa`LPkLG^M##WSE796)5TtxoGghSA8S3q!Sxlz){9b8};XNjtCV#NVMuZ5Dl(!UO}qpDJa& zRDNDo{CkSz!eDd1Va*h}umnnGz4~_VA~%!oAt)67JJI61=@tOws@R&!s-v?#E>Gd( z&s<~PT#dJpN!T?WT%OYIjPYFwZ6B>!_8h5FNNWtr_lKUh5WgMGRskk+K8#{xn8Fbl z?WP2=N+bMlN5S)uVK8Dn4az3c2dJO%-;0b?!h60wV5xkCzln5Hz(i-`2^V6<$w)GI zP#)~h2VcPYUWzgO7%Qtlsspw$o+?`ER5p!LmLqJ;sR!efVXtqkcb^~x#;_P-f&~2_ zP|I0_8u^ny%je8qL}-)`2Wuv|vPESJBO9grc^6<1-|7zFWYa~sk_f)gW$54{IP1N% zkTcUEJN*4Ie$R{Lw7-;$0p^##S=skQsDEbgoTn?D%jm)?{3cUlyYfJhFo4xc==$C( zJ?(0vJa%)o6pMt_Mnh02a>o?)=w*b)TzWaHBZ)kzjzjYBd<#s|(*DDVKzZ9cg#l>3 z^ygv>(;A$;?u?XNdzhA=J)qq=x1a=OS_kjmyV2MP4UG|9)tw!^6KLx-k`_Kc*ikk| z*eD;B03~mJ>$K-+H8)Wf>bme(Q0Ks4u7G(MLC}Ov1$ihB%Eh`PKYXYjCx1~W$(MLo zK0Qb3rW4OTmc}v{`PzI3G+87Y*9W~yiOn3($1MOSo8Bf#9Ft$lcpk2A`%TYU zB&>v7ARYtr^Jju~DXoqk?s})bG%(k`-46iNvn9XKwI!!ke{|`xbp%-6+Q}fCS6gby zdx_jnpXj7UV;7`duQCAGtk>PTy%E3XsE~xG`qSyoF5!pcnwECaqwlRB$nqw0!2;@G zk=NBaw``%A4D{dF<$uYM8`g4xySv0Jr|X<_6Yx)8thb!*#S~JaTd1oD*W4T4IFpaS zGgKOw1I0)Z@dt2s6K|&*2#>RSe?GNIPiF@WDam7{8^yC88~LkEjATUM?gCroUU$<; zt@z63j_^e#COR))0lFNI9%~5W5=e_c`Fg>e^#bT}UsLvrJm=nwo!8^)L?Gdx`np-PQyD(_U)(Zd@%2Z*g?pjfPvZ`E(e!_FYBdX5l!FU zHxH%k;GiuxP%K{fm){w0eG_SuiqL-LycHkloKnNebkpy|a%8KBoS9k;wi##56xokmSv_1n=U5{Lsz`P9kk|NPq;$SQtnYH|AbN#VtR-hplJ zEScYOy51UmqTIx+NJ zH+H6ureaV(BcChq>NTf3@)QK%k07_-b{}TjES|#T*cD01p6VATV(5LCV`@y6=#L01 z%gxr!&nIrY^Komd))CbKKMf!Dw>Bi(2s;i+#H7c=(;Cj$nSxU6+!QOlNU?bAIqSj5~RN^Scc}M@axv4 z{*5@BfUAUn?x0mQk-ebkCvG9w04T?gm^_Ouh7&ANjaXtwq%y*7cLnO66eff_R z1`UPlePpxr=Qx*48N3a6+8I`4^?y(=atZUBp6N;bb8~e^-x}=wzd+p%2XnQll#fTY zzA(~>H_y&vI3t3IAiqCjAH}_|-Q?6q95xH}1&r07UcbCoD9B+eDprjjB2#7d>66vS zQW0F{A1hBZ>x_kT$`)Ado}u>^$_ohJU+yUeeeQzntpQ({^mL8)hQkY;3r!{S*O1#A zsk%2mvtXPXQKtu1ZU`2X!(KX7K%p2}3N zbnt5vMxQ~A?h6fZ*HfG~)MAnOZl6;D<1{J(m-WrSq7>TYT7+ZNi%+t9Xnwv!plx@j zvcm0|1p!>LDe`{v_e2@pmu}X%mVD+)+0x$Ved1&1h=sDqEKjI3Tu8;-R;q)qfbzyR zE~OWQ6<`Q#|2F?OEZTC#yYBjr(@!_jq9Z}ab$=ymaw>gm>^9DKQ)4wq4nk`-Ke&`c z#3%D}hqip2X9G2Eb()KTtnA>R*LM0w_i{bztV~KuI(-)sFL9?ajOi>g%Zw z%jJpm80*&jq;<&6&V8+%Y|3gOW_hk~A6rt| zru-f-0TV2WfGd?JvmUGJNK2W|De5dH7#H_Dg^J^-*(OWV?atBHz04h2gRU7-ug|uC zE^y$frKQ407bTNYlPCI&%~+lRlZNKF`3F#~Os904xY{d{{p78K@fWF4tVmp{vCd@k z*f*WDR0L&3jDlojaqX7;GkUGyg`Z@j#)e{KJUrSJoXl)Ufc{U`AiPkep1%90*LXRu zaqhtWR@E+19fNyJ+m}4WW}J%Wh&a737%SLB2lmdWMM?H10GHR-P*d!!$~ad+C@?Ka zO%eQzL3OwNdFf^T1$>;0&NgY;OVY-W1h*+MqheqMyc4ct+7JgNWOSh6lBYK=_8(Yq z4uOd_PH~(~_|df5;Vhg~`CqVrT5u}re6M-il@vKO7fc;mXEO_(XWfC5Lq}e!)#L|S zpQq1p)o32&3;f7o;O42XD<6&sA?%5z&+JdFMv}<=|NR-@{{jdD_xxb~HCh-!kY*0U zup7?=eRJGjqI|+x6yyoM0nYg(!`u?z_&l^QfFKCQ#RXFRQ0=;t0U8}s9}zLk?bA1) z3rIr?b07$UM5Y&heUN8&Cx#ocs4Ec%CgsoD?`{9^(R2+2K@g0KLsNvaY&}nUCDnn5 zxnwnoa=AT!)0rTOAP9n(^5m6(v^o1c?}R^IHFi?{DSh9c2>9`IkV;!&ngoI%2y)~> zJS4QmW)@aom86Do7=-l9DHdVGgJ@OdlTvK@fDY zM|Ae*yWKDQSJ6zFRP=lpW`=R$zb}Qe2I%91AP9mkzTlSvE{;eQJw`-qw60R4-EcZ+ zwIg3BQ&9v#5Tph!w!0e(V-%5TB5p~QBBsFB%i$0XOB5Q4AP9ogxI)_xCj(YG@?oR_ z7LN{qa%S6lUi3>TBVyxE3bl6aF@G4spa_B>NZn9eG$H}AA)w72 zw}ikyl}7i|*<~4=bex(sRs)q2!dcx ztkE)mTnJro(A$_LOs)o|K?6F_bC93_+R)(*Z+OES-s<57^PV9B{^eYVLaigv?e3FZ zZlCUQ`*BgI~7^DCI002ovPDHLkV1gJ>$xZ+O diff --git a/docs/guides/testing/testing.rst b/docs/guides/testing/testing.rst index a2c62cf45c..9baf24aee7 100644 --- a/docs/guides/testing/testing.rst +++ b/docs/guides/testing/testing.rst @@ -9,8 +9,7 @@ Testing Overview ======== -We maintain three kinds of tests: unit tests, integration tests, and -acceptance tests. +We maintain two kinds of tests: unit tests and integration tests. Overall, you want to write the tests that **maximize coverage** while **minimizing maintenance**. In practice, this usually means investing @@ -22,8 +21,8 @@ the code base. Test Pyramid -The pyramid above shows the relative number of unit tests, integration -tests, and acceptance tests. Most of our tests are unit tests or +The pyramid above shows the relative number of unit tests and integration +tests. Most of our tests are unit tests or integration tests. Test Types @@ -67,19 +66,6 @@ Integration Tests .. _Django test client: https://docs.djangoproject.com/en/dev/topics/testing/overview/ - -UI Acceptance Tests -~~~~~~~~~~~~~~~~~~~ - -- There should be very few UI acceptance tests since they are generally slow and - flaky. Use these to test only bare minimum happy paths for necessary features. - -- We use `Bok Choy`_ to write end-user acceptance tests directly in Python, - using the framework to maximize reliability and maintainability. - -.. _Bok Choy: https://bok-choy.readthedocs.org/en/latest/tutorial.html - - Test Locations -------------- @@ -94,14 +80,6 @@ Test Locations the test for ``src/views/module.js`` should be written in ``spec/views/module_spec.js``. -- UI acceptance tests: - - - Set up and helper methods, and stubs for external services: - ``common/djangoapps/terrain`` - - Bok Choy Acceptance Tests: located under ``common/test/acceptance/tests`` - - Bok Choy Accessibility Tests: located under ``common/test/acceptance/tests`` and tagged with ``@attr("a11y")`` - - Bok Choy PageObjects: located under ``common/test/acceptance/pages`` - Running Tests ============= @@ -109,8 +87,7 @@ You can run all of the unit-level tests using this command:: paver test -This includes python, JavaScript, and documentation tests. It does not, -however, run any acceptance tests. +This includes python, JavaScript, and documentation tests. Note - `paver` is a scripting tool. To get information about various options, you can run the this command:: @@ -310,226 +287,6 @@ Note: the port is also output to the console that you ran the tests from if you These paver commands call through to Karma. For more info, see `karma-runner.github.io `__. -Running Bok Choy Acceptance Tests ---------------------------------- - -We use `Bok Choy`_ for acceptance testing. Bok Choy is a UI-level acceptance -test framework for writing robust `Selenium`_ tests in `Python`_. Bok Choy -makes your acceptance tests reliable and maintainable by utilizing the Page -Object and Promise design patterns. - -**Prerequisites**: - -These prerequisites are all automatically installed and available in -`Devstack`_, the supported development enviornment for the Open edX platform. - -* Chromedriver and Chrome - -* Mongo - -* Memcache - -* mySQL - -To run all the bok choy acceptance tests run this command:: - - paver test_bokchoy - -Once the database has been set up and the static files collected, you -can use the 'fast' option to skip those tasks. This option can also be -used with any of the test specs below:: - - paver test_bokchoy --fasttest - -For example to run a single test, specify the name of the test file:: - - paver test_bokchoy -t lms/test_lms.py - -Notice the test file location is relative to -common/test/acceptance/tests. This is another example:: - - paver test_bokchoy -t studio/test_studio_bad_data.py - -To run a single test faster by not repeating setup tasks use the ``--fasttest`` option:: - - paver test_bokchoy -t studio/test_studio_bad_data.py --fasttest - -To test only a certain feature, specify the file and the testcase class:: - - paver test_bokchoy -t studio/test_studio_bad_data.py::BadComponentTest - -To execute only a certain test case, specify the file name, class, and -test case method:: - - paver test_bokchoy -t lms/test_lms.py::RegistrationTest::test_register - -During acceptance test execution, log files and also screenshots of -failed tests are captured in test\_root/log. - -Use this command to put a temporary debugging breakpoint in a test. -If you check this in, your tests will hang on jenkins:: - - breakpoint() - -By default, all bokchoy tests are run with the 'split' ModuleStore. To -override the modulestore that is used, use the default\_store option. -The currently supported stores are: 'split' -(xmodule.modulestore.split\_mongo.split\_draft.DraftVersioningModuleStore) -and 'draft' (xmodule.modulestore.mongo.DraftMongoModuleStore). This is an example -for the 'draft' store:: - - paver test_bokchoy --default_store='draft' - -Running Bok Choy Accessibility Tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -We use Bok Choy for `automated accessibility testing`_. Bok Choy, a UI-level -acceptance test framework for writing robust `Selenium`_ tests in `Python`_, -includes the ability to perform accessibility audits on web pages using `Google -Accessibility Developer Tools`_ or `Deque's aXe Core`_. For more details about -how to write accessibility tests, please read the `Bok Choy documentation`_ and -the `Automated Accessibility Tests`_ Open edX Confluence page. - -.. _automated accessibility testing: https://bok-choy.readthedocs.org/en/latest/accessibility.html -.. _Selenium: http://docs.seleniumhq.org/ -.. _Python: https://www.python.org/ -.. _Google Accessibility Developer Tools: https://github.com/GoogleChrome/accessibility-developer-tools/ -.. _Deque's aXe Core: https://github.com/dequelabs/axe-core/ -.. _Bok Choy documentation: https://bok-choy.readthedocs.org/en/latest/accessibility.html -.. _Automated Accessibility Tests: https://openedx.atlassian.net/wiki/display/TE/Automated+Accessibility+Tests - - -**Prerequisites**: - -These prerequisites are all automatically installed and available in -`Devstack`_ (since the Cypress release), the supported development environment -for the Open edX platform. - -.. _Devstack: https://github.com/edx/configuration/wiki/edX-Developer-Stack - -* Mongo - -* Memcache - -* mySQL - -To run all the bok choy accessibility tests use this command:: - - paver test_a11y - -To run specific tests, use the ``-t`` flag to specify a pytest-style test spec -relative to the ``common/test/acceptance/tests`` directory. This is an example for it:: - - paver test_a11y -t lms/test_lms_dashboard.py::LmsDashboardA11yTest::test_dashboard_course_listings_a11y - -**Coverage**: - -To generate the coverage report for the views run during accessibility tests:: - - paver a11y_coverage - -Note that this coverage report is just a guideline to find areas that -are missing tests. If the view isn't 'covered', there definitely -isn't a test for it. If it is 'covered', we are loading that page -during the tests but not necessarily calling ``page.a11y_audit.check_for_accessibility_errors`` on it. - - -Options for Faster Development Cycles in Bok-Choy Tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following are ways in which a developer could shorten the development -cycle for faster feedback. The options below can often be used together. - -**Multiprocessing Mode** - -Bok-choy tests can be threaded using the `-n` switch. Using 2 threads generally -reduces test cycles by 33%. The recommendation is to make sure the -number of threads is no more than the number of processors available. For -example, the Cypress release of devstack is provisioned by default with 2 -processors. In that case, to run tests in multiprocess mode:: - - paver test_bokchoy -n 2 - -*Caveat*: Not all tests have been designed with multiprocessing in mind; some -testcases (approx 10%) will fail in multiprocess mode for various reasons -(e.g., shared fixtures, unexpected state, etc). If you have tests that fail -in multiprocessing mode, it may be worthwhile to run them in single-stream mode -to understand if you are encountering such a failure. With that noted, this -can speed development for most test classes. - -**Leave Your Servers Running** - -There are two additional switches available in the `paver test_bokchoy` task. -Used together, they can shorten the cycle between test runs. Similar to above, -there are a handful of tests that won't work with this approach, due to insufficient -teardown and other unmanaged state. - -1. Start your servers in one terminal/ssh session:: - - paver test_bokchoy --serversonly - - Note if setup has already been done, you can run:: - - paver test_bokchoy --serversonly --fasttest - -2. Run your tests only in another terminal/ssh session:: - - paver test_bokchoy --testsonly --fasttest - -You must run BOTH `--testsonly` and `--fasttest`. - -3. When done, you can kill your servers in the first terminal/ssh session with -Control-C. *Warning*: Only hit Control-C one time so the pytest framework can -properly clean up. - -Acceptance Test Techniques -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -1. **Element existence on the page**: Do not use splinter's built-in browser - methods directly for determining if elements exist. Use the - world.is\_css\_present and world.is\_css\_not\_present wrapper - functions instead. Otherwise errors can arise if checks for the css - are performed before the page finishes loading. Also these wrapper - functions are optimized for the amount of wait time spent in both - cases of positive and negative expectation. - -2. **Dealing with alerts**: Chrome can hang on javascripts alerts. If a - javascript alert/prompt/confirmation is expected, use the step 'I - will confirm all alerts', 'I will cancel all alerts' or 'I will anser - all prompts with "(.\*)"' before the step that causes the alert in - order to properly deal with it. - -3. **Dealing with stale element reference exceptions**: These exceptions - happen if any part of the page is refreshed in between finding an - element and accessing the element. When possible, use any of the css - functions in common/djangoapps/terrain/ui\_helpers.py as they will - retry the action in case of this exception. If the functionality is - not there, wrap the function with world.retry\_on\_exception. This - function takes in a function and will retry and return the result of - the function if there was an exception. - -4. **Scenario Level Constants**: If you want an object to be available for - the entire scenario, it can be stored in world.scenario\_dict. This - object is a dictionary that gets refreshed at the beginning on the - scenario. Currently, the current logged in user and the current - created course are stored under 'COURSE' and 'USER'. This will help - prevent strings from being hard coded so the acceptance tests can - become more flexible. - -5. **Internal edX Jenkins considerations**: Acceptance tests are run in - Jenkins as part of the edX development workflow. They are broken into - shards and split across workers. Therefore if you add a new .feature - file, you need to define what shard they should be run in or else - they will not get executed. See someone from TestEng to help you - determine where they should go. - - Also, the test results are rolled up in Jenkins for ease of - understanding, with the acceptance tests under the top level of "CMS" - and "LMS" when they follow this convention: name your feature in the - .feature file CMS or LMS with a single period and then no other - periods in the name. The name can contain spaces. E.g. "CMS.Sign Up" - - Testing internationalization with dummy translations ---------------------------------------------------- @@ -645,7 +402,7 @@ Other Testing Tips Connecting to Browser --------------------- -If you want to see the browser being automated for JavaScript or bok-choy tests, +If you want to see the browser being automated for JavaScript, you can connect to the container running it via VNC. +------------------------+----------------------+ From 7d2a8340c5e997bf4ca43ca537a91e8f0962df1f Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 24 Mar 2022 09:59:53 -0400 Subject: [PATCH 25/52] feat: Update how some settings are read in production.py settings files. Most settings in the production.py files fall back to their values in common.py if they aren't set in the yaml config files but some historically didn't. Update them so that they are more in-line with the rest of the settings in this file. --- cms/envs/production.py | 18 +++++++++--------- lms/envs/production.py | 16 ++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cms/envs/production.py b/cms/envs/production.py index 28895ba74f..dcbf702014 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -170,7 +170,7 @@ ENTERPRISE_CONSENT_API_URL = ENV_TOKENS.get('ENTERPRISE_CONSENT_API_URL', LMS_IN # Studio. Only applies to IDA for which the social auth flow uses DOT (Django OAuth Toolkit). IDA_LOGOUT_URI_LIST = ENV_TOKENS.get('IDA_LOGOUT_URI_LIST', []) -SITE_NAME = ENV_TOKENS['SITE_NAME'] +SITE_NAME = ENV_TOKENS.get('SITE_NAME', SITE_NAME) ALLOWED_HOSTS = [ # TODO: bbeggs remove this before prod, temp fix to get load testing running @@ -178,10 +178,10 @@ ALLOWED_HOSTS = [ CMS_BASE, ] -LOG_DIR = ENV_TOKENS['LOG_DIR'] +LOG_DIR = ENV_TOKENS.get('LOG_DIR', LOG_DIR) DATA_DIR = path(ENV_TOKENS.get('DATA_DIR', DATA_DIR)) -CACHES = ENV_TOKENS['CACHES'] +CACHES = ENV_TOKENS.get('CACHES', CACHES) # Cache used for location mapping -- called many times with the same key/value # in a given request. if 'loc_cache' not in CACHES: @@ -274,7 +274,7 @@ for app in ENV_TOKENS.get('ADDL_INSTALLED_APPS', []): INSTALLED_APPS.append(app) LOGGING = get_logger_config(LOG_DIR, - logging_env=ENV_TOKENS['LOGGING_ENV'], + logging_env=ENV_TOKENS.get('LOGGING_ENV', LOGGING_ENV), service_variant=SERVICE_VARIANT) # The following variables use (or) instead of the default value inside (get). This is to enforce using the Lazy Text @@ -315,11 +315,11 @@ CMS_SEGMENT_KEY = AUTH_TOKENS.get('SEGMENT_KEY') SECRET_KEY = AUTH_TOKENS['SECRET_KEY'] -AWS_ACCESS_KEY_ID = AUTH_TOKENS["AWS_ACCESS_KEY_ID"] +AWS_ACCESS_KEY_ID = AUTH_TOKENS.get("AWS_ACCESS_KEY_ID", AWS_ACCESS_KEY_ID) if AWS_ACCESS_KEY_ID == "": AWS_ACCESS_KEY_ID = None -AWS_SECRET_ACCESS_KEY = AUTH_TOKENS["AWS_SECRET_ACCESS_KEY"] +AWS_SECRET_ACCESS_KEY = AUTH_TOKENS.get("AWS_SECRET_ACCESS_KEY", AWS_SECRET_ACCESS_KEY) if AWS_SECRET_ACCESS_KEY == "": AWS_SECRET_ACCESS_KEY = None @@ -357,7 +357,7 @@ if COURSE_METADATA_EXPORT_BUCKET: else: COURSE_METADATA_EXPORT_STORAGE = DEFAULT_FILE_STORAGE -DATABASES = AUTH_TOKENS['DATABASES'] +DATABASES = AUTH_TOKENS.get('DATABASES', DATABASES) # The normal database user does not have enough permissions to run migrations. # Migrations are run with separate credentials, given as DB_MIGRATION_* @@ -385,8 +385,8 @@ XBLOCK_FIELD_DATA_WRAPPERS = ENV_TOKENS.get( XBLOCK_FIELD_DATA_WRAPPERS ) -CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE'] -DOC_STORE_CONFIG = AUTH_TOKENS['DOC_STORE_CONFIG'] +CONTENTSTORE = AUTH_TOKENS.get('CONTENTSTORE', CONTENTSTORE) +DOC_STORE_CONFIG = AUTH_TOKENS.get('DOC_STORE_CONFIG', DOC_STORE_CONFIG) ############################### BLOCKSTORE ##################################### BLOCKSTORE_API_URL = ENV_TOKENS.get('BLOCKSTORE_API_URL', None) # e.g. "https://blockstore.example.com/api/v1/" diff --git a/lms/envs/production.py b/lms/envs/production.py index 1e54dca63a..e74cff6b58 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -150,7 +150,7 @@ EMAIL_FILE_PATH = ENV_TOKENS.get('EMAIL_FILE_PATH', None) EMAIL_HOST = ENV_TOKENS.get('EMAIL_HOST', 'localhost') # django default is localhost EMAIL_PORT = ENV_TOKENS.get('EMAIL_PORT', 25) # django default is 25 EMAIL_USE_TLS = ENV_TOKENS.get('EMAIL_USE_TLS', False) # django default is False -SITE_NAME = ENV_TOKENS['SITE_NAME'] +SITE_NAME = ENV_TOKENS.get('SITE_NAME', SITE_NAME) SESSION_COOKIE_DOMAIN = ENV_TOKENS.get('SESSION_COOKIE_DOMAIN') SESSION_COOKIE_HTTPONLY = ENV_TOKENS.get('SESSION_COOKIE_HTTPONLY', True) @@ -205,7 +205,7 @@ if ENV_TOKENS.get('SESSION_COOKIE_NAME', None): # By default, it's set to the same thing as the SESSION_COOKIE_DOMAIN, but we want to make it overrideable. SHARED_COOKIE_DOMAIN = ENV_TOKENS.get('SHARED_COOKIE_DOMAIN', SESSION_COOKIE_DOMAIN) -CACHES = ENV_TOKENS['CACHES'] +CACHES = ENV_TOKENS.get('CACHES', CACHES) # Cache used for location mapping -- called many times with the same key/value # in a given request. if 'loc_cache' not in CACHES: @@ -315,11 +315,11 @@ for app in ENV_TOKENS.get('ADDL_INSTALLED_APPS', []): local_loglevel = ENV_TOKENS.get('LOCAL_LOGLEVEL', 'INFO') -LOG_DIR = ENV_TOKENS['LOG_DIR'] +LOG_DIR = ENV_TOKENS.get('LOG_DIR', LOG_DIR) DATA_DIR = path(ENV_TOKENS.get('DATA_DIR', DATA_DIR)) LOGGING = get_logger_config(LOG_DIR, - logging_env=ENV_TOKENS['LOGGING_ENV'], + logging_env=ENV_TOKENS.get('LOGGING_ENV', LOGGING_ENV), local_loglevel=local_loglevel, service_variant=SERVICE_VARIANT) @@ -444,11 +444,11 @@ LMS_SEGMENT_KEY = AUTH_TOKENS.get('SEGMENT_KEY') SECRET_KEY = AUTH_TOKENS['SECRET_KEY'] -AWS_ACCESS_KEY_ID = AUTH_TOKENS["AWS_ACCESS_KEY_ID"] +AWS_ACCESS_KEY_ID = AUTH_TOKENS.get("AWS_ACCESS_KEY_ID", AWS_ACCESS_KEY_ID) if AWS_ACCESS_KEY_ID == "": AWS_ACCESS_KEY_ID = None -AWS_SECRET_ACCESS_KEY = AUTH_TOKENS["AWS_SECRET_ACCESS_KEY"] +AWS_SECRET_ACCESS_KEY = AUTH_TOKENS.get("AWS_SECRET_ACCESS_KEY", AWS_SECRET_ACCESS_KEY) if AWS_SECRET_ACCESS_KEY == "": AWS_SECRET_ACCESS_KEY = None @@ -468,7 +468,7 @@ else: # If there is a database called 'read_replica', you can use the use_read_replica_if_available # function in util/query.py, which is useful for very large database reads -DATABASES = AUTH_TOKENS['DATABASES'] +DATABASES = AUTH_TOKENS.get('DATABASES', DATABASES) # The normal database user does not have enough permissions to run migrations. # Migrations are run with separate credentials, given as DB_MIGRATION_* @@ -484,7 +484,7 @@ for name, database in DATABASES.items(): 'PORT': os.environ.get('DB_MIGRATION_PORT', database['PORT']), }) -XQUEUE_INTERFACE = AUTH_TOKENS['XQUEUE_INTERFACE'] +XQUEUE_INTERFACE = AUTH_TOKENS.get('XQUEUE_INTERFACE', XQUEUE_INTERFACE) # Get the MODULESTORE from auth.json, but if it doesn't exist, # use the one from common.py From f4c3471a9bc49b17f5c145f9b3fc5f80ba4a84a5 Mon Sep 17 00:00:00 2001 From: Alexander Sheehan Date: Thu, 24 Mar 2022 10:23:21 -0400 Subject: [PATCH 26/52] fix: adding was_valid_at to all provider configs --- .../migrations/0008_auto_20220324_1422.py | 23 +++++++++++++++++++ common/djangoapps/third_party_auth/models.py | 16 ++++++------- .../third_party_auth/tests/test_admin.py | 3 ++- 3 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 common/djangoapps/third_party_auth/migrations/0008_auto_20220324_1422.py diff --git a/common/djangoapps/third_party_auth/migrations/0008_auto_20220324_1422.py b/common/djangoapps/third_party_auth/migrations/0008_auto_20220324_1422.py new file mode 100644 index 0000000000..653a9287db --- /dev/null +++ b/common/djangoapps/third_party_auth/migrations/0008_auto_20220324_1422.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.12 on 2022-03-24 14:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('third_party_auth', '0007_samlproviderconfig_was_valid_at'), + ] + + operations = [ + migrations.AddField( + model_name='ltiproviderconfig', + name='was_valid_at', + field=models.DateTimeField(blank=True, help_text='Timestamped field that indicates a user has successfully logged in using this configuration at least once.', null=True), + ), + migrations.AddField( + model_name='oauth2providerconfig', + name='was_valid_at', + field=models.DateTimeField(blank=True, help_text='Timestamped field that indicates a user has successfully logged in using this configuration at least once.', null=True), + ), + ] diff --git a/common/djangoapps/third_party_auth/models.py b/common/djangoapps/third_party_auth/models.py index 9686a756e2..f0a37616a3 100644 --- a/common/djangoapps/third_party_auth/models.py +++ b/common/djangoapps/third_party_auth/models.py @@ -223,6 +223,14 @@ class ProviderConfig(ConfigurationModel): ) ) + was_valid_at = models.DateTimeField( + blank=True, + null=True, + help_text=( + "Timestamped field that indicates a user has successfully logged in using this configuration at least once." + ) + ) + prefix = None # used for provider_id. Set to a string value in subclass backend_name = None # Set to a field or fixed value in subclass accepts_logins = True # Whether to display a sign-in button when the provider is enabled @@ -700,14 +708,6 @@ class SAMLProviderConfig(ProviderConfig): blank=True, ) - was_valid_at = models.DateTimeField( - blank=True, - null=True, - help_text=( - "Timestamped field that indicates a user has successfully logged in using this configuration at least once." - ) - ) - def clean(self): """ Standardize and validate fields """ super().clean() diff --git a/common/djangoapps/third_party_auth/tests/test_admin.py b/common/djangoapps/third_party_auth/tests/test_admin.py index 82a93e8db8..c5481a3a09 100644 --- a/common/djangoapps/third_party_auth/tests/test_admin.py +++ b/common/djangoapps/third_party_auth/tests/test_admin.py @@ -64,10 +64,11 @@ class Oauth2ProviderConfigAdminTest(testutil.TestCase): # Remove the icon_image from the POST data, to simulate unchanged icon_image post_data = models.model_to_dict(provider1) del post_data['icon_image'] - # Remove max_session_length and organization. A default null value must be POSTed + # Remove max_session_length, was_valid_at and organization. A default null value must be POSTed # back as an absent value, rather than as a "null-like" included value. del post_data['max_session_length'] del post_data['organization'] + del post_data['was_valid_at'] # Change the name, to verify POST post_data['name'] = 'Another name' From df22dfbe0df23da821e3b483f5a7861f30c265c7 Mon Sep 17 00:00:00 2001 From: Shafqat Farhan Date: Fri, 25 Mar 2022 05:49:48 +0500 Subject: [PATCH 27/52] feat: VAN-669 - Disallow bad passwords on Registration --- cms/envs/common.py | 10 ++++++++++ lms/envs/common.py | 10 ++++++++++ openedx/core/djangoapps/user_api/accounts/api.py | 13 +++++++++---- .../user_authn/views/registration_form.py | 12 +++++++++++- 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/cms/envs/common.py b/cms/envs/common.py index fe1dbb0a08..6c3e01894d 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -518,6 +518,16 @@ LIBRARY_AUTHORING_MICROFRONTEND_URL = None # .. toggle_creation_date: 2021-12-03 # .. toggle_tickets: https://openedx.atlassian.net/browse/VAN-666 ENABLE_AUTHN_RESET_PASSWORD_HIBP_POLICY = False +# .. toggle_name: ENABLE_AUTHN_REGISTER_HIBP_POLICY +# .. toggle_implementation: DjangoSetting +# .. toggle_default: False +# .. toggle_description: When enabled, this toggle activates the use of the password validation +# HIBP Policy on Authn MFE's registration. +# .. toggle_use_cases: open_edx +# .. toggle_creation_date: 2022-03-25 +# .. toggle_tickets: https://openedx.atlassian.net/browse/VAN-669 +ENABLE_AUTHN_REGISTER_HIBP_POLICY = False +HIBP_REGISTRATION_PASSWORD_FREQUENCY_THRESHOLD = 3 ############################# SOCIAL MEDIA SHARING ############################# SOCIAL_SHARING_SETTINGS = { diff --git a/lms/envs/common.py b/lms/envs/common.py index 876dde9d0c..22e52102ab 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -4809,6 +4809,16 @@ DISCUSSIONS_MFE_FEEDBACK_URL = None # .. toggle_creation_date: 2021-12-03 # .. toggle_tickets: https://openedx.atlassian.net/browse/VAN-666 ENABLE_AUTHN_RESET_PASSWORD_HIBP_POLICY = False +# .. toggle_name: ENABLE_AUTHN_REGISTER_HIBP_POLICY +# .. toggle_implementation: DjangoSetting +# .. toggle_default: False +# .. toggle_description: When enabled, this toggle activates the use of the password validation +# HIBP Policy on Authn MFE's registration. +# .. toggle_use_cases: open_edx +# .. toggle_creation_date: 2022-03-25 +# .. toggle_tickets: https://openedx.atlassian.net/browse/VAN-669 +ENABLE_AUTHN_REGISTER_HIBP_POLICY = False +HIBP_REGISTRATION_PASSWORD_FREQUENCY_THRESHOLD = 3 ############### Settings for the ace_common plugin ################# ACE_ENABLED_CHANNELS = ['django_email'] diff --git a/openedx/core/djangoapps/user_api/accounts/api.py b/openedx/core/djangoapps/user_api/accounts/api.py index 4fa9e76563..754882f2ce 100644 --- a/openedx/core/djangoapps/user_api/accounts/api.py +++ b/openedx/core/djangoapps/user_api/accounts/api.py @@ -641,12 +641,17 @@ def _validate_password(password, username=None, email=None, reset_password_page= except ValidationError as validation_err: raise errors.AccountPasswordInvalid(' '.join(validation_err.messages)) - # TODO: VAN-666 - Restrict this feature to reset password page for now until it is - # enabled on account sign in and register. - if settings.ENABLE_AUTHN_RESET_PASSWORD_HIBP_POLICY and reset_password_page: + if ( + (settings.ENABLE_AUTHN_RESET_PASSWORD_HIBP_POLICY and reset_password_page) or + (settings.ENABLE_AUTHN_REGISTER_HIBP_POLICY and not reset_password_page) + ): pwned_response = check_pwned_password(password) if pwned_response.get('vulnerability', 'no') == 'yes': - raise errors.AccountPasswordInvalid(accounts.AUTHN_PASSWORD_COMPROMISED_MSG) + if ( + reset_password_page or + pwned_response.get('frequency', 0) >= settings.HIBP_REGISTRATION_PASSWORD_FREQUENCY_THRESHOLD + ): + raise errors.AccountPasswordInvalid(accounts.AUTHN_PASSWORD_COMPROMISED_MSG) def _validate_country(country): diff --git a/openedx/core/djangoapps/user_authn/views/registration_form.py b/openedx/core/djangoapps/user_authn/views/registration_form.py index 79a66fa554..0321250a34 100644 --- a/openedx/core/djangoapps/user_authn/views/registration_form.py +++ b/openedx/core/djangoapps/user_authn/views/registration_form.py @@ -21,7 +21,7 @@ from common.djangoapps.edxmako.shortcuts import marketing_link from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers from openedx.core.djangoapps.user_api import accounts from openedx.core.djangoapps.user_api.helpers import FormDescription -from openedx.core.djangoapps.user_authn.utils import is_registration_api_v1 as is_api_v1 +from openedx.core.djangoapps.user_authn.utils import check_pwned_password, is_registration_api_v1 as is_api_v1 from openedx.core.djangolib.markup import HTML, Text from openedx.features.enterprise_support.api import enterprise_customer_for_request from common.djangoapps.student.models import ( @@ -238,6 +238,16 @@ class AccountCreationForm(forms.Form): email = self.cleaned_data.get('email') temp_user = User(username=username, email=email) if username else None validate_password(password, temp_user) + + if settings.ENABLE_AUTHN_REGISTER_HIBP_POLICY: + # Checks the Pwned Databases for password vulnerability. + pwned_response = check_pwned_password(password) + + if ( + pwned_response.get('vulnerability', 'no') == 'yes' and + pwned_response.get('frequency', 0) >= settings.HIBP_REGISTRATION_PASSWORD_FREQUENCY_THRESHOLD + ): + raise ValidationError(accounts.AUTHN_PASSWORD_COMPROMISED_MSG) return password def clean_email(self): From bc45f1ee48980edaacee7663cf0b847ba2aaed66 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Fri, 25 Mar 2022 15:27:39 +0500 Subject: [PATCH 28/52] fix: Removed usage of djangoratelimitbackend. (#30116) * fix: Removed usage of djangoratelimitbackend. --- cms/djangoapps/contentstore/tests/test_admin.py | 12 ------------ cms/djangoapps/contentstore/views/public.py | 7 +------ cms/templates/admin/base_site.html | 7 ++----- common/djangoapps/util/request_rate_limiter.py | 4 ---- lms/templates/admin/base_site.html | 8 ++------ .../oauth_dispatch/dot_overrides/backends.py | 3 --- .../core/djangoapps/user_authn/config/waffle.py | 15 +-------------- .../core/djangoapps/user_authn/views/login.py | 11 ++--------- openedx/core/tests/test_admin_view.py | 16 +--------------- requirements/edx-sandbox/py38.txt | 2 +- requirements/edx/base.txt | 15 ++++++++------- requirements/edx/development.txt | 11 +++++------ requirements/edx/github.in | 4 ---- requirements/edx/testing.txt | 11 +++++------ 14 files changed, 28 insertions(+), 98 deletions(-) delete mode 100644 common/djangoapps/util/request_rate_limiter.py delete mode 100644 openedx/core/djangoapps/oauth_dispatch/dot_overrides/backends.py diff --git a/cms/djangoapps/contentstore/tests/test_admin.py b/cms/djangoapps/contentstore/tests/test_admin.py index d0a306ba11..43e3c23369 100644 --- a/cms/djangoapps/contentstore/tests/test_admin.py +++ b/cms/djangoapps/contentstore/tests/test_admin.py @@ -6,9 +6,6 @@ This is not inside a django app because it is a global property of the system. import ddt from django.test import TestCase from django.urls import reverse -from edx_toggles.toggles.testutils import override_waffle_flag - -from openedx.core.djangoapps.user_authn.config.waffle import ADMIN_AUTH_REDIRECT_TO_LMS @ddt.ddt @@ -16,17 +13,8 @@ class TestAdminView(TestCase): """ Tests of the admin view. """ - @override_waffle_flag(ADMIN_AUTH_REDIRECT_TO_LMS, True) @ddt.data('/admin/', '/admin/login', reverse('admin:login')) def test_admin_login_redirect(self, admin_url): """Admin login will redirect towards the site login page.""" response = self.client.get(admin_url, follow=True) assert any('/login/edx-oauth2/?next=' in r[0] for r in response.redirect_chain) - - def test_admin_login_default(self): - """Without flag Admin login will redirect towards the admin default login page.""" - response = self.client.get('/admin/', follow=True) - assert response.status_code == 200 - self.assertIn('/admin/login/?next=/admin/', response.redirect_chain[0]) - assert len(response.redirect_chain) == 1 - assert response.template_name == ['admin/login.html'] diff --git a/cms/djangoapps/contentstore/views/public.py b/cms/djangoapps/contentstore/views/public.py index 823bcd4154..2d5e69bc23 100644 --- a/cms/djangoapps/contentstore/views/public.py +++ b/cms/djangoapps/contentstore/views/public.py @@ -7,10 +7,8 @@ from django.conf import settings from django.shortcuts import redirect from urllib.parse import quote_plus # lint-amnesty, pylint: disable=wrong-import-order from waffle.decorators import waffle_switch -from django.contrib import admin from common.djangoapps.edxmako.shortcuts import render_to_response -from openedx.core.djangoapps.user_authn.config.waffle import ADMIN_AUTH_REDIRECT_TO_LMS from ..config import waffle @@ -48,10 +46,7 @@ def redirect_to_lms_login_for_admin(request): """ This view redirect the admin/login url to the site's login page. """ - if ADMIN_AUTH_REDIRECT_TO_LMS.is_enabled(): - return redirect('/login?next=/admin') - else: - return admin.site.login(request) + return redirect('/login?next=/admin') def _build_next_param(request): diff --git a/cms/templates/admin/base_site.html b/cms/templates/admin/base_site.html index dd10b68f5f..0783178430 100644 --- a/cms/templates/admin/base_site.html +++ b/cms/templates/admin/base_site.html @@ -17,10 +17,7 @@ {% endif %} {% endif %} - {% flag "user_authn.admin_auth_redirect_to_lms" %} - {% trans 'Log out' as tmsg %} {{tmsg|force_escape}} - {% else %} - {% trans 'Log out' as tmsg %} {{tmsg|force_escape}} - {% endflag %} + {% trans 'Log out' as tmsg %} {{tmsg|force_escape}} + {% endblock %} diff --git a/common/djangoapps/util/request_rate_limiter.py b/common/djangoapps/util/request_rate_limiter.py deleted file mode 100644 index a9e9b6312e..0000000000 --- a/common/djangoapps/util/request_rate_limiter.py +++ /dev/null @@ -1,4 +0,0 @@ -""" -A utility class which wraps the RateLimitMixin 3rd party class to do bad request counting -which can be used for rate limiting -""" diff --git a/lms/templates/admin/base_site.html b/lms/templates/admin/base_site.html index 285d275282..4ea8630769 100644 --- a/lms/templates/admin/base_site.html +++ b/lms/templates/admin/base_site.html @@ -1,6 +1,5 @@ {% extends "admin/base.html" %} {% load i18n admin_urls %} -{% load waffle_tags %} {% block title %}{{ title }} | {{ site_title|default:_('Django site admin') }}{% endblock %} {% block branding %}

{{ site_header|default:_('Django administration') }}

@@ -17,10 +16,7 @@ {% endif %} {% endif %} - {% flag "user_authn.admin_auth_redirect_to_lms" %} - {% trans 'Log out' as tmsg%}{{tmsg|force_escape}} - {% else %} - {% trans 'Log out' as tmsg%}{{tmsg|force_escape}} - {% endflag %} + {% trans 'Log out' as tmsg%}{{tmsg|force_escape}} + {% endblock %} diff --git a/openedx/core/djangoapps/oauth_dispatch/dot_overrides/backends.py b/openedx/core/djangoapps/oauth_dispatch/dot_overrides/backends.py deleted file mode 100644 index d115a3a5ec..0000000000 --- a/openedx/core/djangoapps/oauth_dispatch/dot_overrides/backends.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -Custom authentication backends. -""" diff --git a/openedx/core/djangoapps/user_authn/config/waffle.py b/openedx/core/djangoapps/user_authn/config/waffle.py index c58b81869d..dc409d4eaa 100644 --- a/openedx/core/djangoapps/user_authn/config/waffle.py +++ b/openedx/core/djangoapps/user_authn/config/waffle.py @@ -3,7 +3,7 @@ Waffle flags and switches for user authn. """ -from edx_toggles.toggles import LegacyWaffleSwitch, LegacyWaffleSwitchNamespace, WaffleFlag +from edx_toggles.toggles import LegacyWaffleSwitch, LegacyWaffleSwitchNamespace _WAFFLE_NAMESPACE = 'user_authn' _WAFFLE_SWITCH_NAMESPACE = LegacyWaffleSwitchNamespace(name=_WAFFLE_NAMESPACE, log_prefix='UserAuthN: ') @@ -37,16 +37,3 @@ ENABLE_PWNED_PASSWORD_API = LegacyWaffleSwitch( 'enable_pwned_password_api', __name__ ) - - -# .. toggle_name: ADMIN_AUTH_REDIRECT_TO_LMS -# .. toggle_implementation: WaffleFlag -# .. toggle_default: False -# .. toggle_description: Set this to True if you want to redirect cms-admin login to lms login. -# In case of logout it will use lms logout also. -# .. toggle_use_cases: open_edx -# .. toggle_creation_date: 2022-02-08 -# .. toggle_target_removal_date: None -ADMIN_AUTH_REDIRECT_TO_LMS = WaffleFlag( # lint-amnesty, pylint: disable=toggle-missing-annotation - "user_authn.admin_auth_redirect_to_lms", module_name=__name__ -) diff --git a/openedx/core/djangoapps/user_authn/views/login.py b/openedx/core/djangoapps/user_authn/views/login.py index 7ad123c9eb..537ab32ec3 100644 --- a/openedx/core/djangoapps/user_authn/views/login.py +++ b/openedx/core/djangoapps/user_authn/views/login.py @@ -11,7 +11,6 @@ import re import urllib from django.conf import settings -from django.contrib import admin from django.contrib.auth import authenticate, get_user_model from django.contrib.auth import login as django_login from django.contrib.auth.decorators import login_required @@ -43,10 +42,7 @@ from common.djangoapps.util.password_policy_validators import normalize_password from openedx.core.djangoapps.password_policy import compliance as password_policy_compliance from openedx.core.djangoapps.safe_sessions.middleware import mark_user_change_as_expected from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers -from openedx.core.djangoapps.user_authn.config.waffle import ( - ENABLE_LOGIN_USING_THIRDPARTY_AUTH_ONLY, - ADMIN_AUTH_REDIRECT_TO_LMS -) +from openedx.core.djangoapps.user_authn.config.waffle import ENABLE_LOGIN_USING_THIRDPARTY_AUTH_ONLY from openedx.core.djangoapps.user_authn.cookies import get_response_with_refreshed_jwt_cookies, set_logged_in_cookies from openedx.core.djangoapps.user_authn.exceptions import AuthFailedError from openedx.core.djangoapps.user_authn.toggles import ( @@ -658,10 +654,7 @@ def redirect_to_lms_login(request): This view redirect the admin/login url to the site's login page if waffle switch is on otherwise returns the admin site's login view. """ - if ENABLE_LOGIN_USING_THIRDPARTY_AUTH_ONLY.is_enabled() or ADMIN_AUTH_REDIRECT_TO_LMS.is_enabled(): - return redirect('/login?next=/admin') - else: - return admin.site.login(request) + return redirect('/login?next=/admin') class LoginSessionView(APIView): diff --git a/openedx/core/tests/test_admin_view.py b/openedx/core/tests/test_admin_view.py index b0a70ac965..1eafb9f0de 100644 --- a/openedx/core/tests/test_admin_view.py +++ b/openedx/core/tests/test_admin_view.py @@ -6,9 +6,8 @@ This is not inside a django app because it is a global property of the system. from django.test import Client, TestCase from django.urls import reverse -from edx_toggles.toggles.testutils import override_waffle_switch, override_waffle_flag +from edx_toggles.toggles.testutils import override_waffle_switch from common.djangoapps.student.tests.factories import UserFactory, TEST_PASSWORD -from openedx.core.djangoapps.user_authn.config.waffle import ADMIN_AUTH_REDIRECT_TO_LMS from openedx.core.djangoapps.user_authn.views.login import ENABLE_LOGIN_USING_THIRDPARTY_AUTH_ONLY @@ -44,16 +43,3 @@ class TestAdminView(TestCase): response = self.client.get(reverse('admin:login')) assert response.url == '/login?next=/admin' assert response.status_code == 302 - - with override_waffle_flag(ADMIN_AUTH_REDIRECT_TO_LMS, True): - response = self.client.get(reverse('admin:login')) - assert response.url == '/login?next=/admin' - assert response.status_code == 302 - - with override_waffle_switch(ENABLE_LOGIN_USING_THIRDPARTY_AUTH_ONLY, False): - response = self.client.get(reverse('admin:login')) - assert response.template_name == ['admin/login.html'] - - with override_waffle_flag(ADMIN_AUTH_REDIRECT_TO_LMS, False): - response = self.client.get(reverse('admin:login')) - assert response.template_name == ['admin/login.html'] diff --git a/requirements/edx-sandbox/py38.txt b/requirements/edx-sandbox/py38.txt index 0fc9c65db9..482679a6b7 100644 --- a/requirements/edx-sandbox/py38.txt +++ b/requirements/edx-sandbox/py38.txt @@ -89,5 +89,5 @@ sympy==1.6.2 # -c requirements/edx-sandbox/../constraints.txt # -r requirements/edx-sandbox/py38.in # openedx-calc -tqdm==4.63.0 +tqdm==4.63.1 # via nltk diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 1a1eafe3a1..c40ce8a7c4 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -47,7 +47,9 @@ appdirs==1.4.4 asgiref==3.5.0 # via django async-timeout==4.0.2 - # via aiohttp + # via + # aiohttp + # redis attrs==21.4.0 # via # -r requirements/edx/base.in @@ -195,7 +197,6 @@ django==3.2.12 # django-mysql # django-oauth-toolkit # django-pyfs - # django-ratelimit-backend # django-sekizai # django-ses # django-splash @@ -330,8 +331,6 @@ django-pyfs==3.2.0 # via -r requirements/edx/base.in django-ratelimit==3.0.1 # via -r requirements/edx/base.in -django-ratelimit-backend @ git+https://github.com/edx/django-ratelimit-backend.git@6e1a0c6ea1d27062c16e9fb94d3c44475146877e - # via -r requirements/edx/github.in django-require @ git+https://github.com/edx/django-require.git@0c54adb167142383b26ea6b3edecc3211822a776 # via -r requirements/edx/github.in django-sekizai==3.0.1 @@ -913,7 +912,7 @@ ruamel-yaml==0.17.21 # via drf-yasg ruamel-yaml-clib==0.2.6 # via ruamel-yaml -rules==3.2.1 +rules==3.3 # via # -r requirements/edx/base.in # edx-enterprise @@ -1019,10 +1018,12 @@ testfixtures==6.18.5 # via edx-enterprise text-unidecode==1.3 # via python-slugify -tqdm==4.63.0 +tqdm==4.63.1 # via nltk typing-extensions==4.1.1 - # via django-countries + # via + # django-countries + # redis unicodecsv==0.14.1 # via # -r requirements/edx/base.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index ca40f56961..bc4b64d251 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -74,6 +74,7 @@ async-timeout==4.0.2 # via # -r requirements/edx/testing.txt # aiohttp + # redis attrs==21.4.0 # via # -r requirements/edx/testing.txt @@ -279,7 +280,6 @@ django==3.2.12 # django-mysql # django-oauth-toolkit # django-pyfs - # django-ratelimit-backend # django-sekizai # django-ses # django-splash @@ -426,8 +426,6 @@ django-pyfs==3.2.0 # via -r requirements/edx/testing.txt django-ratelimit==3.0.1 # via -r requirements/edx/testing.txt -django-ratelimit-backend @ git+https://github.com/edx/django-ratelimit-backend.git@6e1a0c6ea1d27062c16e9fb94d3c44475146877e - # via -r requirements/edx/testing.txt django-require @ git+https://github.com/edx/django-require.git@0c54adb167142383b26ea6b3edecc3211822a776 # via -r requirements/edx/testing.txt django-sekizai==3.0.1 @@ -660,7 +658,7 @@ execnet==1.9.0 # pytest-xdist factory-boy==3.2.1 # via -r requirements/edx/testing.txt -faker==13.3.2 +faker==13.3.3 # via # -r requirements/edx/testing.txt # factory-boy @@ -1279,7 +1277,7 @@ ruamel-yaml-clib==0.2.6 # via # -r requirements/edx/testing.txt # ruamel-yaml -rules==3.2.1 +rules==3.3 # via # -r requirements/edx/testing.txt # edx-enterprise @@ -1471,7 +1469,7 @@ tox==3.24.5 # tox-battery tox-battery==0.6.1 # via -r requirements/edx/testing.txt -tqdm==4.63.0 +tqdm==4.63.1 # via # -r requirements/edx/testing.txt # nltk @@ -1485,6 +1483,7 @@ typing-extensions==4.1.1 # mypy # pydantic # pylint + # redis unicodecsv==0.14.1 # via # -r requirements/edx/testing.txt diff --git a/requirements/edx/github.in b/requirements/edx/github.in index 5e126a4d56..50bfa1da6c 100644 --- a/requirements/edx/github.in +++ b/requirements/edx/github.in @@ -59,10 +59,6 @@ git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2 -e git+https://github.com/jazkarta/edx-jsme.git@690dbf75441fa91c7c4899df0b83d77f7deb5458#egg=edx-jsme -# This is a temporary fork until https://github.com/brutasse/django-ratelimit-backend/pull/50 is merged -# back into the upstream code. -git+https://github.com/edx/django-ratelimit-backend.git@6e1a0c6ea1d27062c16e9fb94d3c44475146877e#egg=django-ratelimit-backend - # original repo is not maintained any more. git+https://github.com/edx/django-require.git@0c54adb167142383b26ea6b3edecc3211822a776#egg=django-require==1.0.12 diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index daddddfee0..aa3f700c0f 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -69,6 +69,7 @@ async-timeout==4.0.2 # via # -r requirements/edx/base.txt # aiohttp + # redis attrs==21.4.0 # via # -r requirements/edx/base.txt @@ -267,7 +268,6 @@ distlib==0.3.4 # django-mysql # django-oauth-toolkit # django-pyfs - # django-ratelimit-backend # django-sekizai # django-ses # django-splash @@ -412,8 +412,6 @@ django-pyfs==3.2.0 # via -r requirements/edx/base.txt django-ratelimit==3.0.1 # via -r requirements/edx/base.txt -django-ratelimit-backend @ git+https://github.com/edx/django-ratelimit-backend.git@6e1a0c6ea1d27062c16e9fb94d3c44475146877e - # via -r requirements/edx/base.txt django-require @ git+https://github.com/edx/django-require.git@0c54adb167142383b26ea6b3edecc3211822a776 # via -r requirements/edx/base.txt django-sekizai==3.0.1 @@ -641,7 +639,7 @@ execnet==1.9.0 # via pytest-xdist factory-boy==3.2.1 # via -r requirements/edx/testing.in -faker==13.3.2 +faker==13.3.3 # via factory-boy fastapi==0.75.0 # via pact-python @@ -1203,7 +1201,7 @@ ruamel-yaml-clib==0.2.6 # via # -r requirements/edx/base.txt # ruamel-yaml -rules==3.2.1 +rules==3.3 # via # -r requirements/edx/base.txt # edx-enterprise @@ -1359,7 +1357,7 @@ tox==3.24.5 # tox-battery tox-battery==0.6.1 # via -r requirements/edx/testing.in -tqdm==4.63.0 +tqdm==4.63.1 # via # -r requirements/edx/base.txt # nltk @@ -1372,6 +1370,7 @@ typing-extensions==4.1.1 # django-countries # pydantic # pylint + # redis unicodecsv==0.14.1 # via # -r requirements/edx/base.txt From e35c7e278bf6496732a3a4df2be81ffd2bc63ed1 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 24 Mar 2022 10:46:42 -0400 Subject: [PATCH 29/52] build: Simplify how we build docs. Instead of trying to use the devstack settings file for building docs, use the common settings file as the base for all settings. The devstack settings file builds on top of the production settings file which are both oriented around reading settings from a yaml file and getting them loaded in sanely into the dev and production environment. For documentation, start with the common settings files which should be sufficient to get a default version of the system up and running. Note: We still leave the loop that enables all the feature flags as a way of finding conditionally included API endpoints. --- docs/docs_settings.py | 14 +++++++++++--- docs/guides/conf.py | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/docs_settings.py b/docs/docs_settings.py index 3b6df8c41a..8c1b813df1 100644 --- a/docs/docs_settings.py +++ b/docs/docs_settings.py @@ -7,8 +7,10 @@ import all the Studio code. import os -from lms.envs.devstack import * # lint-amnesty, pylint: disable=wildcard-import -from cms.envs.devstack import ( # lint-amnesty, pylint: disable=unused-import +from openedx.core.lib.derived import derive_settings + +from lms.envs.common import * # lint-amnesty, pylint: disable=wildcard-import +from cms.envs.common import ( # lint-amnesty, pylint: disable=unused-import ADVANCED_PROBLEM_TYPES, COURSE_IMPORT_EXPORT_STORAGE, GIT_EXPORT_DEFAULT_IDENT, @@ -16,6 +18,7 @@ from cms.envs.devstack import ( # lint-amnesty, pylint: disable=unused-import SCRAPE_YOUTUBE_THUMBNAILS_JOB_QUEUE, VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE, UPDATE_SEARCH_INDEX_JOB_QUEUE, + FRONTEND_REGISTER_URL, ) # Turn on all the boolean feature flags, so that conditionally included @@ -26,13 +29,18 @@ for key, value in FEATURES.items(): # Settings that will fail if we enable them, and we don't need them for docs anyway. FEATURES['RUN_AS_ANALYTICS_SERVER_ENABLED'] = False +FEATURES['ENABLE_SOFTWARE_SECURE_FAKE'] = False +FEATURES['ENABLE_MKTG_SITE'] = False INSTALLED_APPS.extend([ 'cms.djangoapps.contentstore.apps.ContentstoreConfig', 'cms.djangoapps.course_creators', 'cms.djangoapps.xblock_config.apps.XBlockConfig', - 'lms.djangoapps.lti_provider' + 'lms.djangoapps.lti_provider', + 'user_tasks', ]) COMMON_TEST_DATA_ROOT = '' + +derive_settings(__name__) diff --git a/docs/guides/conf.py b/docs/guides/conf.py index f3038eb462..213d131ce3 100644 --- a/docs/guides/conf.py +++ b/docs/guides/conf.py @@ -235,7 +235,7 @@ def update_settings_module(service='lms'): Set the "DJANGO_SETTINGS_MODULE" environment variable appropriately for the module sphinx-apidoc is about to be run on. """ - if os.environ['EDX_PLATFORM_SETTINGS'] == 'devstack_docker': + if os.environ.get('EDX_PLATFORM_SETTINGS') == 'devstack_docker': settings_module = f'{service}.envs.devstack_docker' else: settings_module = f'{service}.envs.devstack' From 31d3fcc01a3a6279fd3477c7416c9560f00ce8e7 Mon Sep 17 00:00:00 2001 From: Justin Hynes Date: Tue, 22 Mar 2022 15:12:33 -0400 Subject: [PATCH 30/52] feat: add models for scheduled instructor tasks [MICROBA-1508] - Adds the InstructorTaskSchedule model - Adds the HistoricalInstructorTaskSchedule model - Adds utility function used to create a InstructorTaskSchedule instance - Adds a public `create_course_email` Python API function to the bulk_email app that can be imported and used in external apps - Adds a new `test_api_helper.py` test file (with tests for the new `schedule_task` function) to the instructor_task app - Adds a new `test_api.py` test file (with tests for the new `create_course_email` function to the bulk_email app --- lms/djangoapps/bulk_email/api.py | 41 +++++- lms/djangoapps/bulk_email/data.py | 22 +++ lms/djangoapps/bulk_email/tests/test_api.py | 96 +++++++++++++ lms/djangoapps/instructor/views/api.py | 5 +- lms/djangoapps/instructor_task/api_helper.py | 45 +++++- ...ctortaskschedule_instructortaskschedule.py | 55 +++++++ lms/djangoapps/instructor_task/models.py | 24 ++++ .../instructor_task/tests/test_api.py | 2 + .../instructor_task/tests/test_api_helper.py | 134 ++++++++++++++++++ 9 files changed, 421 insertions(+), 3 deletions(-) create mode 100644 lms/djangoapps/bulk_email/data.py create mode 100644 lms/djangoapps/bulk_email/tests/test_api.py create mode 100644 lms/djangoapps/instructor_task/migrations/0004_historicalinstructortaskschedule_instructortaskschedule.py create mode 100644 lms/djangoapps/instructor_task/tests/test_api_helper.py diff --git a/lms/djangoapps/bulk_email/api.py b/lms/djangoapps/bulk_email/api.py index 5c2e40ee64..f89a27065b 100644 --- a/lms/djangoapps/bulk_email/api.py +++ b/lms/djangoapps/bulk_email/api.py @@ -4,11 +4,12 @@ Python APIs exposed by the bulk_email app to other in-process apps. """ # Public Bulk Email Functions - +import logging from django.conf import settings from django.urls import reverse +from lms.djangoapps.bulk_email.models import CourseEmail from lms.djangoapps.bulk_email.models_api import ( is_bulk_email_disabled_for_course, is_bulk_email_enabled_for_course, @@ -18,6 +19,8 @@ from lms.djangoapps.bulk_email.models_api import ( from lms.djangoapps.discussion.notification_prefs.views import UsernameCipher from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers +log = logging.getLogger(__name__) + def get_emails_enabled(user, course_id): """ @@ -48,3 +51,39 @@ def get_unsubscribed_link(username, course_id): optout_url = reverse('bulk_email_opt_out', kwargs={'token': token, 'course_id': course_id}) url = f'{lms_root_url}{optout_url}' return url + + +def create_course_email(course_id, sender, targets, subject, html_message, text_message=None, template_name=None, + from_addr=None): + """ + Python API for creating a new CourseEmail instance. + + Args: + course_id (CourseKey): The CourseKey of the course. + sender (String): Email author. + targets (Target): Recipient groups the message should be sent to (e.g. SEND_TO_MYSELF) + subject (String)): Email subject. + html_message (String): Email body. Includes HTML markup. + text_message (String, optional): Plaintext version of email body. Defaults to None. + template_name (String, optional): Name of custom email template to use. Defaults to None. + from_addr (String, optional): Custom sending address, if desired. Defaults to None. + + Returns: + CourseEmail: Returns the created CourseEmail instance. + """ + try: + course_email = CourseEmail.create( + course_id, + sender, + targets, + subject, + html_message, + text_message=text_message, + template_name=template_name, + from_addr=from_addr + ) + + return course_email + except ValueError as err: + log.exception(f"Cannot create course email for {course_id} requested by user {sender} for targets {targets}") + raise ValueError from err diff --git a/lms/djangoapps/bulk_email/data.py b/lms/djangoapps/bulk_email/data.py new file mode 100644 index 0000000000..a0f50c6c6b --- /dev/null +++ b/lms/djangoapps/bulk_email/data.py @@ -0,0 +1,22 @@ +""" +Bulk Email Data + +This provides Data models to represent Bulk Email data. +""" + + +class BulkEmailTargetChoices: + """ + Enum for the available targets (recipient groups) of an email authored with the bulk course email tool. + + SEND_TO_MYSELF - Message intended for author of the message + SEND_TO_STAFF - Message intended for all course staff + SEND_TO_LEARNERS - Message intended for all enrolled learners + SEND_TO_COHORT - Message intended for a specific cohort + SEND_TO_TRACK - Message intended for all learners in a specific track (e.g. audit or verified) + """ + SEND_TO_MYSELF = "myself" + SEND_TO_STAFF = "staff" + SEND_TO_LEARNERS = "learners" + SEND_TO_COHORT = "cohort" + SEND_TO_TRACK = "track" diff --git a/lms/djangoapps/bulk_email/tests/test_api.py b/lms/djangoapps/bulk_email/tests/test_api.py new file mode 100644 index 0000000000..f6c0804724 --- /dev/null +++ b/lms/djangoapps/bulk_email/tests/test_api.py @@ -0,0 +1,96 @@ +""" +Tests for the public Python API functions of the Bulk Email app. +""" +from testfixtures import LogCapture + +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory + +from common.djangoapps.student.tests.factories import InstructorFactory +from lms.djangoapps.bulk_email.api import create_course_email +from lms.djangoapps.bulk_email.data import BulkEmailTargetChoices +from openedx.core.lib.html_to_text import html_to_text + + +class CreateCourseEmailTests(ModuleStoreTestCase): + """ + Tests for the `create_course_email` function of the bulk email app's public Python API. + """ + def setUp(self): + super().setUp() + self.course = CourseFactory.create() + self.instructor = InstructorFactory(course_key=self.course.id) + self.target = [BulkEmailTargetChoices.SEND_TO_MYSELF] + self.subject = "email subject" + self.html_message = "

test message

" + + def test_create_course_email(self): + """ + Happy path test for the `create_course_email` function. Verifies the creation of a CourseEmail instance with + the bare minimum information required for the function call. + """ + course_email = create_course_email( + self.course.id, + self.instructor, + self.target, + self.subject, + self.html_message, + ) + + assert course_email.sender.id == self.instructor.id + assert course_email.subject == self.subject + assert course_email.html_message == self.html_message + assert course_email.course_id == self.course.id + assert course_email.text_message == html_to_text(self.html_message) + + def test_create_course_email_with_optional_args(self): + """ + Additional testing to verify that optional data is used as expected when passed into the `create_course_email` + function. + """ + text_message = "everything is awesome!" + template_name = "gnarly_template" + from_addr = "blub@noreply.fish.com" + + course_email = create_course_email( + self.course.id, + self.instructor, + self.target, + self.subject, + self.html_message, + text_message=text_message, + template_name=template_name, + from_addr=from_addr + ) + + assert course_email.sender.id == self.instructor.id + assert course_email.subject == self.subject + assert course_email.html_message == self.html_message + assert course_email.course_id == self.course.id + assert course_email.text_message == text_message + assert course_email.template_name == template_name + assert course_email.from_addr == from_addr + + def test_create_course_email_expect_exception(self): + """ + Test to verify behavior when an exception occurs when calling teh `create_course_email` function. + """ + targets = ["humpty dumpty"] + + expected_messages = [ + f"Cannot create course email for {self.course.id} requested by user {self.instructor} for targets " + f"{targets}", + ] + + with self.assertRaises(ValueError): + with LogCapture() as log: + create_course_email( + self.course.id, + self.instructor, + targets, + self.subject, + self.html_message + ) + + for index, message in enumerate(expected_messages): + assert message in log.records[index].getMessage() diff --git a/lms/djangoapps/instructor/views/api.py b/lms/djangoapps/instructor/views/api.py index 67cc72b68b..81b9b6b487 100644 --- a/lms/djangoapps/instructor/views/api.py +++ b/lms/djangoapps/instructor/views/api.py @@ -2733,12 +2733,15 @@ def send_email(request, course_id): # Create the CourseEmail object. This is saved immediately, so that # any transaction that has been pending up to this point will also be # committed. + # TODO: convert to use bulk_email app's `create_course_email` API function and remove direct import and use of + # bulk_email model try: email = CourseEmail.create( course_id, request.user, targets, - subject, message, + subject, + message, template_name=template_name, from_addr=from_addr ) diff --git a/lms/djangoapps/instructor_task/api_helper.py b/lms/djangoapps/instructor_task/api_helper.py index 59096b91d0..506833e6de 100644 --- a/lms/djangoapps/instructor_task/api_helper.py +++ b/lms/djangoapps/instructor_task/api_helper.py @@ -17,7 +17,7 @@ from opaque_keys.edx.keys import UsageKey from common.djangoapps.util.db import outer_atomic from lms.djangoapps.courseware.courses import get_problems_in_section -from lms.djangoapps.instructor_task.models import PROGRESS, InstructorTask +from lms.djangoapps.instructor_task.models import PROGRESS, SCHEDULED, InstructorTask, InstructorTaskSchedule from xmodule.modulestore.django import modulestore # lint-amnesty, pylint: disable=wrong-import-order log = logging.getLogger(__name__) @@ -448,3 +448,46 @@ def submit_task(request, task_type, task_class, course_key, task_input, task_key _handle_instructor_task_failure(instructor_task, error) return instructor_task + + +def schedule_task(request, task_type, course_key, task_input, task_key, schedule): + """ + Helper function to schedule a background task. + + Reserves the requested task and stores it until the task is ready for execution. We also create an instance of a + InstructorTaskSchedule object responsible for maintaining the details of _when_ a task should be executed. Extracts + arguments important to the task from the originating server request and stores them as part of the schedule object. + Sets the `task_status` to SCHEDULED to indicate this task will be executed in the future. + + Args: + request (WSGIRequest): The originating web request associated with this task request. + task_type (String): Text describing the type of task (e.g. 'bulk_course_email' or 'grade_course') + course_key (CourseKey): The CourseKey of the course-run the task belongs to. + task_input (dict): Task input arguments stores as JSON-serialized dictionary. + task_key (String): Encoded input arguments used during task execution. + schedule (DateTime): DateTime (in UTC) describing when the task should be executed. + """ + instructor_task = None + try: + log.info(f"Creating a scheduled instructor task of type '{task_type}' for course '{course_key}' requested by " + f"user with id '{request.user.id}'") + instructor_task = InstructorTask.create(course_key, task_type, task_key, task_input, request.user) + + task_id = instructor_task.task_id + task_args = _get_xmodule_instance_args(request, task_id) + log.info(f"Creating a task schedule associated with instructor task '{instructor_task.id}' and due after " + f"'{schedule}'") + InstructorTaskSchedule.objects.create( + task=instructor_task, + task_args=json.dumps(task_args), + task_due=schedule, + ) + + log.info(f"Updating task state of instructor task '{instructor_task.id}' to '{SCHEDULED}'") + instructor_task.task_state = SCHEDULED + instructor_task.save() + except Exception as error: # pylint: disable=broad-except + log.error(f"Error occurred during task or schedule creation: {error}") + # Set any orphaned instructor tasks to the FAILURE state. + if instructor_task: + _handle_instructor_task_failure(instructor_task, error) diff --git a/lms/djangoapps/instructor_task/migrations/0004_historicalinstructortaskschedule_instructortaskschedule.py b/lms/djangoapps/instructor_task/migrations/0004_historicalinstructortaskschedule_instructortaskschedule.py new file mode 100644 index 0000000000..36b6ddb186 --- /dev/null +++ b/lms/djangoapps/instructor_task/migrations/0004_historicalinstructortaskschedule_instructortaskschedule.py @@ -0,0 +1,55 @@ +# Generated by Django 3.2.12 on 2022-03-22 18:49 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import model_utils.fields +import simple_history.models + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('instructor_task', '0003_alter_task_input_field'), + ] + + operations = [ + migrations.CreateModel( + name='InstructorTaskSchedule', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('task_args', models.TextField()), + ('task_due', models.DateTimeField()), + ('task', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='instructor_task.instructortask')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='HistoricalInstructorTaskSchedule', + fields=[ + ('id', models.IntegerField(auto_created=True, blank=True, db_index=True, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('task_args', models.TextField()), + ('task_due', models.DateTimeField()), + ('history_id', models.AutoField(primary_key=True, serialize=False)), + ('history_date', models.DateTimeField()), + ('history_change_reason', models.CharField(max_length=100, null=True)), + ('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)), + ('history_user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)), + ('task', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='instructor_task.instructortask')), + ], + options={ + 'verbose_name': 'historical instructor task schedule', + 'ordering': ('-history_date', '-history_id'), + 'get_latest_by': 'history_date', + }, + bases=(simple_history.models.HistoricalChanges, models.Model), + ), + ] diff --git a/lms/djangoapps/instructor_task/models.py b/lms/djangoapps/instructor_task/models.py index 1309ac3283..92268c3090 100644 --- a/lms/djangoapps/instructor_task/models.py +++ b/lms/djangoapps/instructor_task/models.py @@ -20,13 +20,16 @@ import os.path from uuid import uuid4 from boto.exception import BotoServerError +from django.apps import apps from django.conf import settings from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user from django.core.files.base import ContentFile from django.db import models, transaction from django.utils.translation import gettext as _ +from model_utils.models import TimeStampedModel from opaque_keys.edx.django.models import CourseKeyField +from simple_history.models import HistoricalRecords from openedx.core.storage import get_storage @@ -35,6 +38,7 @@ logger = logging.getLogger(__name__) # define custom states used by InstructorTask QUEUING = 'QUEUING' PROGRESS = 'PROGRESS' +SCHEDULED = 'SCHEDULED' TASK_INPUT_LENGTH = 10000 @@ -187,6 +191,26 @@ class InstructorTask(models.Model): return json.dumps({'message': 'Task revoked before running'}) +class InstructorTaskSchedule(TimeStampedModel): + """ + A database model to store information about _when_ to execute a scheduled background task. + + The primary use case is to allow instructors to schedule their email messages (authored with the bulk course email + tool) to be sent at a later date and time. + + .. no_pii: + """ + class Meta: + app_label = "instructor_task" + + task = models.OneToOneField(InstructorTask, on_delete=models.CASCADE) + task_args = models.TextField(null=False, blank=False) + task_due = models.DateTimeField(null=False) + + if 'instructor_task' in apps.app_configs: + history = HistoricalRecords() + + class ReportStore: """ Simple abstraction layer that can fetch and store CSV files for reports diff --git a/lms/djangoapps/instructor_task/tests/test_api.py b/lms/djangoapps/instructor_task/tests/test_api.py index 678291cc28..fd0c842944 100644 --- a/lms/djangoapps/instructor_task/tests/test_api.py +++ b/lms/djangoapps/instructor_task/tests/test_api.py @@ -205,6 +205,8 @@ class InstructorTaskCourseSubmitTest(TestReportMixin, InstructorTaskCourseTestCa def _define_course_email(self): """Create CourseEmail object for testing.""" + # TODO: convert to use bulk_email app's `create_course_email` API function and remove direct import and use of + # bulk_email model course_email = CourseEmail.create( self.course.id, self.instructor, diff --git a/lms/djangoapps/instructor_task/tests/test_api_helper.py b/lms/djangoapps/instructor_task/tests/test_api_helper.py new file mode 100644 index 0000000000..dcba37bb46 --- /dev/null +++ b/lms/djangoapps/instructor_task/tests/test_api_helper.py @@ -0,0 +1,134 @@ +""" +Tests for the Instructor Task `api_helper.py` functions. +""" +import datetime +import hashlib +import json +from unittest.mock import patch +from testfixtures import LogCapture + +from celery.states import FAILURE + +from common.djangoapps.student.tests.factories import UserFactory +from lms.djangoapps.bulk_email.api import create_course_email +from lms.djangoapps.bulk_email.data import BulkEmailTargetChoices +from lms.djangoapps.instructor_task.api_helper import QueueConnectionError, schedule_task +from lms.djangoapps.instructor_task.models import SCHEDULED, InstructorTask, InstructorTaskSchedule +from lms.djangoapps.instructor_task.tests.test_base import InstructorTaskCourseTestCase + + +class ScheduledBulkEmailInstructorTaskTests(InstructorTaskCourseTestCase): + """ + Tests for the `schedule_task` functionality, with a focus on the scheduled bulk email tasks. + """ + class FakeRequest: + """ + Test class reflecting a portion of the properties expected in a WSGIRequest. We use data from the originating + web request during execution of Instructor Tasks. + """ + def __init__(self, user): + self.user = user + self.META = { + "REMOTE_ADDR": "127.0.0.1", + 'HTTP_USER_AGENT': 'test_agent', + 'SERVER_NAME': 'test_server_name', + } + + def setUp(self): + super().setUp() + + self.initialize_course() + self.instructor = UserFactory.create(username="instructor", email="instructor@edx.org") + self.request = self.FakeRequest(self.instructor) + self.targets = [BulkEmailTargetChoices.SEND_TO_MYSELF] + self.course_email = self._create_course_email(self.targets) + self.schedule = datetime.datetime.now(datetime.timezone.utc) + self.task_type = "bulk_course_email" + self.task_input = json.dumps(self._generate_bulk_email_task_input(self.course_email, self.targets)) + self.task_key = hashlib.md5(str(self.course_email.id).encode('utf-8')).hexdigest() + + def _create_course_email(self, targets): + """ + Create CourseEmail object for testing. + """ + course_email = create_course_email( + self.course.id, + self.instructor, + targets, + "Test Subject", + "

Test message.

" + ) + + return course_email + + def _generate_bulk_email_task_input(self, course_email, targets): + return { + "email_id": course_email.id, + "to_option": targets + } + + def _verify_log_messages(self, expected_messages, log): + for index, message in enumerate(expected_messages): + assert message in log.records[index].getMessage() + + def test_create_scheduled_instructor_task(self): + """ + Happy path test for the `schedule_task` function. Verifies that we create an InstructorTask instance and an + associated InstructorTaskSchedule instance as expected. + """ + with LogCapture() as log: + schedule_task(self.request, self.task_type, self.course.id, self.task_input, self.task_key, self.schedule) + + # get the task instance and its associated schedule for verifications + task = InstructorTask.objects.get(course_id=self.course.id, task_key=self.task_key) + task_schedule = InstructorTaskSchedule.objects.get(task=task) + expected_task_args = { + "request_info": { + "username": self.instructor.username, + "user_id": self.instructor.id, + "ip": "127.0.0.1", + "agent": "test_agent", + "host": "test_server_name", + }, + "task_id": task.task_id + } + expected_messages = [ + f"Creating a scheduled instructor task of type '{self.task_type}' for course '{self.course.id}' requested " + f"by user with id '{self.request.user.id}'", + f"Creating a task schedule associated with instructor task '{task.id}' and due after '{self.schedule}'", + f"Updating task state of instructor task '{task.id}' to '{SCHEDULED}'" + ] + # convert from text back to JSON before comparison + actual_task_args = json.loads(task_schedule.task_args) + + # verify the task has the correct state + assert task.task_state == SCHEDULED + # verify that the schedule is associated with the correct task_id (UUID) + assert task_schedule.task_id == task.id + # verify that the schedule is the expected date and time + assert task_schedule.task_due == self.schedule + # verify the task_arguments are as expected + assert expected_task_args == actual_task_args + self._verify_log_messages(expected_messages, log) + + @patch("lms.djangoapps.instructor_task.api_helper._get_xmodule_instance_args", side_effect=Exception("boom!")) + def test_create_scheduled_instructor_task_expect_failure(self, mock_get_xmodule_instance_args): + """ + A test to verify that we will mark a task as `FAILED` if a failure occurs during the creation of the task + schedule. + """ + expected_messages = [ + f"Creating a scheduled instructor task of type '{self.task_type}' for course '{self.course.id}' requested " + f"by user with id '{self.request.user.id}'", + "Error occurred during task or schedule creation: boom!", + ] + + with self.assertRaises(QueueConnectionError): + with LogCapture() as log: + schedule_task( + self.request, self.task_type, self.course.id, self.task_input, self.task_key, self.schedule + ) + + task = InstructorTask.objects.get(course_id=self.course.id, task_key=self.task_key) + assert task.task_state == FAILURE + self._verify_log_messages(expected_messages, log) From c5319d6c2bb163654c82e57b99a729267c6a9e46 Mon Sep 17 00:00:00 2001 From: Syed Sajjad Hussain Shah <52817156+syedsajjadkazmii@users.noreply.github.com> Date: Mon, 28 Mar 2022 12:38:22 +0500 Subject: [PATCH 31/52] fix: Add course key or program UUID (#30125) * fix: Add course key or program UUID Added course key or program uuid for Save For Later segment event VAN-905 * fix: fixed failing test * fix: fixed PEP 8 failure --- lms/djangoapps/save_for_later/helper.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/lms/djangoapps/save_for_later/helper.py b/lms/djangoapps/save_for_later/helper.py index 7ff0975e46..03f89b5811 100644 --- a/lms/djangoapps/save_for_later/helper.py +++ b/lms/djangoapps/save_for_later/helper.py @@ -113,14 +113,23 @@ def send_email(request, email, data): }] braze_client.track_user(events=[event_properties], attributes=attributes) + + event_data = { + 'user_id': request.user.id, + 'category': 'save-for-later', + 'type': event_properties.get('type'), + 'send_to_self': bool(not request.user.is_anonymous and request.user.email == email), + } + if data.get('type') == 'program': + program = data.get('program') + event_data.update({'program_uuid': program.get('uuid')}) + elif data.get('type') == 'course': + course = data.get('course') + event_data.update({'course_key': str(course.id)}) + tracker.emit( USER_SENT_EMAIL_SAVE_FOR_LATER, - { - 'user_id': request.user.id, - 'category': 'save-for-later', - 'type': event_properties.get('type'), - 'send_to_self': bool(not request.user.is_anonymous and request.user.email == email), - } + event_data ) except Exception: # pylint: disable=broad-except log.warning('Unable to send save for later email ', exc_info=True) From edd7e0a77d7289bff2cc3962782644e10d822725 Mon Sep 17 00:00:00 2001 From: Julia Eskew Date: Fri, 25 Mar 2022 12:11:32 -0400 Subject: [PATCH 32/52] feat: Combine the two existing log messages for submitting courses to Neo4j for viewing in coursegraph for two reasons. First, the existing log message was misleading as it implied that a course was being submitted to Neo4j even when it was being skipped due to not changing since last being sent to Neo4j. Second, the new log message was not distinctive enough for separate searching in Splunk - it will now be distinctive enough for that search. --- openedx/core/djangoapps/coursegraph/tasks.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/openedx/core/djangoapps/coursegraph/tasks.py b/openedx/core/djangoapps/coursegraph/tasks.py index bfca4f79b6..ec943e7527 100644 --- a/openedx/core/djangoapps/coursegraph/tasks.py +++ b/openedx/core/djangoapps/coursegraph/tasks.py @@ -371,13 +371,6 @@ class ModuleStoreSerializer: # first, clear the request cache to prevent memory leaks RequestCache.clear_all_namespaces() - log.info( - "Now submitting %s for export to neo4j: course %d of %d total courses", - course_key, - index + 1, - total_number_of_courses, - ) - (needs_dump, reason) = should_dump_course(course_key, graph) if not (override_cache or needs_dump): log.info("skipping submitting %s, since it hasn't changed", course_key) @@ -386,7 +379,15 @@ class ModuleStoreSerializer: if override_cache: reason = "override_cache is True" - log.info("submitting %s, because %s", course_key, reason) + + log.info( + "Now submitting %s for export to neo4j, because %s: course %d of %d total courses", + course_key, + reason, + index + 1, + total_number_of_courses, + ) + dump_course_to_neo4j.apply_async( args=[str(course_key), credentials], ) From e27a8ad12590cf775637d206c1430ce94594cc7b Mon Sep 17 00:00:00 2001 From: Ghassan Maslamani Date: Mon, 28 Mar 2022 21:57:23 +0300 Subject: [PATCH 33/52] fix: add as style for the link tag that preloads (#30094) Adding a as `as="style"` for link tag which preloads a CSS resource That is requried to set otherwise chrome/browser would issue a warning about it. Ref: https://developer.mozilla.org/en-US/docs/Web/HTML/Link_types/preload --- lms/templates/courseware/courseware-chromeless.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/templates/courseware/courseware-chromeless.html b/lms/templates/courseware/courseware-chromeless.html index df5d9ee1c2..af36cacae5 100644 --- a/lms/templates/courseware/courseware-chromeless.html +++ b/lms/templates/courseware/courseware-chromeless.html @@ -42,7 +42,7 @@ ${static.get_page_title_breadcrumbs(course_name())} - + ${HTML(fragment.head_html())} % if is_learning_mfe: From 103ff55dd1625dfc978a6b8387d50a55a80e763b Mon Sep 17 00:00:00 2001 From: jansenk Date: Mon, 28 Mar 2022 15:54:00 -0400 Subject: [PATCH 34/52] chore: bump ORA to 4.0.6 --- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/testing.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index c40ce8a7c4..6914978507 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -715,7 +715,7 @@ openedx-events==0.8.1 # via -r requirements/edx/base.in openedx-filters==0.5.0 # via -r requirements/edx/base.in -ora2==4.0.5 +ora2==4.0.6 # via -r requirements/edx/base.in packaging==21.3 # via diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index bc4b64d251..a983de0bcd 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -953,7 +953,7 @@ openedx-events==0.8.1 # via -r requirements/edx/testing.txt openedx-filters==0.5.0 # via -r requirements/edx/testing.txt -ora2==4.0.5 +ora2==4.0.6 # via -r requirements/edx/testing.txt packaging==21.3 # via diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index aa3f700c0f..d6f5de95d3 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -901,7 +901,7 @@ openedx-events==0.8.1 # via -r requirements/edx/base.txt openedx-filters==0.5.0 # via -r requirements/edx/base.txt -ora2==4.0.5 +ora2==4.0.6 # via -r requirements/edx/base.txt packaging==21.3 # via From 7c7d7d8b6f1b3867f5ffa1e8f848e515634a382e Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Mon, 28 Mar 2022 20:38:34 +0000 Subject: [PATCH 35/52] feat: Allow REST_FRAMEWORK to be configured by (shallow) merge (#30112) ARCHBOM-2073 --- cms/envs/production.py | 4 ++++ lms/envs/production.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/cms/envs/production.py b/cms/envs/production.py index dcbf702014..8b29e3d3f3 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -83,6 +83,7 @@ with codecs.open(CONFIG_FILE, encoding='utf-8') as f: 'CELERY_QUEUES', 'MKTG_URL_LINK_MAP', 'MKTG_URL_OVERRIDES', + 'REST_FRAMEWORK', ] for key in KEYS_WITH_MERGED_VALUES: if key in __config_copy__: @@ -629,3 +630,6 @@ DISCUSSIONS_MICROFRONTEND_URL = ENV_TOKENS.get('DISCUSSIONS_MICROFRONTEND_URL', ################### Discussions micro frontend Feedback URL################### DISCUSSIONS_MFE_FEEDBACK_URL = ENV_TOKENS.get('DISCUSSIONS_MFE_FEEDBACK_URL', DISCUSSIONS_MFE_FEEDBACK_URL) + +############## DRF overrides ############## +REST_FRAMEWORK.update(ENV_TOKENS.get('REST_FRAMEWORK', {})) diff --git a/lms/envs/production.py b/lms/envs/production.py index e74cff6b58..611a9870fc 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -82,6 +82,7 @@ with codecs.open(CONFIG_FILE, encoding='utf-8') as f: 'CELERY_QUEUES', 'MKTG_URL_LINK_MAP', 'MKTG_URL_OVERRIDES', + 'REST_FRAMEWORK', ] for key in KEYS_WITH_MERGED_VALUES: if key in __config_copy__: @@ -1064,3 +1065,6 @@ DISCUSSIONS_MICROFRONTEND_URL = ENV_TOKENS.get('DISCUSSIONS_MICROFRONTEND_URL', ################### Discussions micro frontend Feedback URL################### DISCUSSIONS_MFE_FEEDBACK_URL = ENV_TOKENS.get('DISCUSSIONS_MFE_FEEDBACK_URL', DISCUSSIONS_MFE_FEEDBACK_URL) + +############## DRF overrides ############## +REST_FRAMEWORK.update(ENV_TOKENS.get('REST_FRAMEWORK', {})) From f1d930fb35d0567b603c4a845e1569eb0fa8f426 Mon Sep 17 00:00:00 2001 From: Kira Miller <31229189+kiram15@users.noreply.github.com> Date: Mon, 28 Mar 2022 18:20:18 -0400 Subject: [PATCH 36/52] chore: version bump (#30136) --- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index d0301dd8bc..961de0338c 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -31,7 +31,7 @@ django-storages<1.9 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==3.41.3 +edx-enterprise==3.41.6 # Newer versions need a more recent version of python-dateutil freezegun==0.3.12 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 6914978507..e7e9a8f463 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -452,7 +452,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.3 +edx-enterprise==3.41.6 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index a983de0bcd..c100cc2f89 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -557,7 +557,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.3 +edx-enterprise==3.41.6 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index d6f5de95d3..f2b91c69b2 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -541,7 +541,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.3 +edx-enterprise==3.41.6 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt From 8039e40f47bdce418480075fee906e8fec896fa9 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 20 Jan 2022 18:46:59 -0500 Subject: [PATCH 37/52] refactor: move coursegraph to cms This code was originally located at: ./openedx/core/djangoapps/coursegraph However, code makes more sense within the ./cms tree, because: * it is responsible for publishing course content to an external system, with is within the responsibilities of CMS, and * is uses modulestore, which is discouraged for use in LMS (see 0011-limit-modulestore-use-in-lms.rst). So, we move the code to: ./cms/djangoapps/coursegraph and uninstall coursegraph from LMS. We do not expect this refactor to have any breaking downstream effects. --- .github/workflows/unit-test-shards.json | 3 +- .../djangoapps/coursegraph/README.rst | 0 .../djangoapps/coursegraph/__init__.py | 0 .../djangoapps/coursegraph/apps.py | 4 +- .../coursegraph/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../management/commands/dump_to_neo4j.py | 2 +- .../management/commands/tests/__init__.py | 0 .../commands/tests/test_dump_to_neo4j.py | 44 +++++++++---------- .../management/commands/tests/utils.py | 0 .../djangoapps/coursegraph/tasks.py | 0 cms/envs/common.py | 2 +- cms/envs/production.py | 2 +- lms/envs/common.py | 4 -- 14 files changed, 28 insertions(+), 33 deletions(-) rename {openedx/core => cms}/djangoapps/coursegraph/README.rst (100%) rename {openedx/core => cms}/djangoapps/coursegraph/__init__.py (100%) rename {openedx/core => cms}/djangoapps/coursegraph/apps.py (65%) rename {openedx/core => cms}/djangoapps/coursegraph/management/__init__.py (100%) rename {openedx/core => cms}/djangoapps/coursegraph/management/commands/__init__.py (100%) rename {openedx/core => cms}/djangoapps/coursegraph/management/commands/dump_to_neo4j.py (97%) rename {openedx/core => cms}/djangoapps/coursegraph/management/commands/tests/__init__.py (100%) rename {openedx/core => cms}/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py (91%) rename {openedx/core => cms}/djangoapps/coursegraph/management/commands/tests/utils.py (100%) rename {openedx/core => cms}/djangoapps/coursegraph/tasks.py (100%) diff --git a/.github/workflows/unit-test-shards.json b/.github/workflows/unit-test-shards.json index f7cd11bf2c..838d0fe261 100644 --- a/.github/workflows/unit-test-shards.json +++ b/.github/workflows/unit-test-shards.json @@ -99,7 +99,6 @@ "openedx/core/djangoapps/course_apps/", "openedx/core/djangoapps/course_date_signals/", "openedx/core/djangoapps/course_groups/", - "openedx/core/djangoapps/coursegraph/", "openedx/core/djangoapps/courseware_api/", "openedx/core/djangoapps/crawlers/", "openedx/core/djangoapps/credentials/", @@ -181,7 +180,6 @@ "openedx/core/djangoapps/course_apps/", "openedx/core/djangoapps/course_date_signals/", "openedx/core/djangoapps/course_groups/", - "openedx/core/djangoapps/coursegraph/", "openedx/core/djangoapps/courseware_api/", "openedx/core/djangoapps/crawlers/", "openedx/core/djangoapps/credentials/", @@ -240,6 +238,7 @@ "paths": [ "cms/djangoapps/api/", "cms/djangoapps/cms_user_tasks/", + "cms/djangoapps/coursegraph/", "cms/djangoapps/course_creators/", "cms/djangoapps/export_course_metadata/", "cms/djangoapps/maintenance/", diff --git a/openedx/core/djangoapps/coursegraph/README.rst b/cms/djangoapps/coursegraph/README.rst similarity index 100% rename from openedx/core/djangoapps/coursegraph/README.rst rename to cms/djangoapps/coursegraph/README.rst diff --git a/openedx/core/djangoapps/coursegraph/__init__.py b/cms/djangoapps/coursegraph/__init__.py similarity index 100% rename from openedx/core/djangoapps/coursegraph/__init__.py rename to cms/djangoapps/coursegraph/__init__.py diff --git a/openedx/core/djangoapps/coursegraph/apps.py b/cms/djangoapps/coursegraph/apps.py similarity index 65% rename from openedx/core/djangoapps/coursegraph/apps.py rename to cms/djangoapps/coursegraph/apps.py index ecedf33d19..95d7873fce 100644 --- a/openedx/core/djangoapps/coursegraph/apps.py +++ b/cms/djangoapps/coursegraph/apps.py @@ -12,6 +12,6 @@ class CoursegraphConfig(AppConfig): """ AppConfig for courseware app """ - name = 'openedx.core.djangoapps.coursegraph' + name = 'cms.djangoapps.coursegraph' - from openedx.core.djangoapps.coursegraph import tasks + from cms.djangoapps.coursegraph import tasks diff --git a/openedx/core/djangoapps/coursegraph/management/__init__.py b/cms/djangoapps/coursegraph/management/__init__.py similarity index 100% rename from openedx/core/djangoapps/coursegraph/management/__init__.py rename to cms/djangoapps/coursegraph/management/__init__.py diff --git a/openedx/core/djangoapps/coursegraph/management/commands/__init__.py b/cms/djangoapps/coursegraph/management/commands/__init__.py similarity index 100% rename from openedx/core/djangoapps/coursegraph/management/commands/__init__.py rename to cms/djangoapps/coursegraph/management/commands/__init__.py diff --git a/openedx/core/djangoapps/coursegraph/management/commands/dump_to_neo4j.py b/cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py similarity index 97% rename from openedx/core/djangoapps/coursegraph/management/commands/dump_to_neo4j.py rename to cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py index 226b3c174a..5fcaefafcb 100644 --- a/openedx/core/djangoapps/coursegraph/management/commands/dump_to_neo4j.py +++ b/cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py @@ -9,7 +9,7 @@ from textwrap import dedent from django.core.management.base import BaseCommand -from openedx.core.djangoapps.coursegraph.tasks import ModuleStoreSerializer +from cms.djangoapps.coursegraph.tasks import ModuleStoreSerializer log = logging.getLogger(__name__) diff --git a/openedx/core/djangoapps/coursegraph/management/commands/tests/__init__.py b/cms/djangoapps/coursegraph/management/commands/tests/__init__.py similarity index 100% rename from openedx/core/djangoapps/coursegraph/management/commands/tests/__init__.py rename to cms/djangoapps/coursegraph/management/commands/tests/__init__.py diff --git a/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py b/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py similarity index 91% rename from openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py rename to cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py index 0d03f56d5f..37d2dfaf66 100644 --- a/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py +++ b/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py @@ -13,10 +13,10 @@ from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory import openedx.core.djangoapps.content.block_structure.config as block_structure_config -from openedx.core.djangoapps.content.block_structure.signals import update_block_structure_on_course_publish -from openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j import ModuleStoreSerializer -from openedx.core.djangoapps.coursegraph.management.commands.tests.utils import MockGraph, MockNodeMatcher -from openedx.core.djangoapps.coursegraph.tasks import ( +from cms.djangoapps.content.block_structure.signals import update_block_structure_on_course_publish +from cms.djangoapps.coursegraph.management.commands.dump_to_neo4j import ModuleStoreSerializer +from cms.djangoapps.coursegraph.management.commands.tests.utils import MockGraph, MockNodeMatcher +from cms.djangoapps.coursegraph.tasks import ( coerce_types, serialize_course, serialize_item, @@ -115,8 +115,8 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): Tests for the dump to neo4j management command """ - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.Graph') + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') + @mock.patch('cms.djangoapps.coursegraph.tasks.Graph') @ddt.data(1, 2) def test_dump_specific_courses(self, number_of_courses, mock_graph_class, mock_matcher_class): """ @@ -140,8 +140,8 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): number_rollbacks=0 ) - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.Graph') + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') + @mock.patch('cms.djangoapps.coursegraph.tasks.Graph') def test_dump_skip_course(self, mock_graph_class, mock_matcher_class): """ Test that you can skip courses. @@ -166,8 +166,8 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): number_rollbacks=0, ) - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.Graph') + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') + @mock.patch('cms.djangoapps.coursegraph.tasks.Graph') def test_dump_skip_beats_specifying(self, mock_graph_class, mock_matcher_class): """ Test that if you skip and specify the same course, you'll skip it. @@ -193,8 +193,8 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): number_rollbacks=0, ) - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.Graph') + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') + @mock.patch('cms.djangoapps.coursegraph.tasks.Graph') def test_dump_all_courses(self, mock_graph_class, mock_matcher_class): """ Test if you don't specify which courses to dump, then you'll dump @@ -395,8 +395,8 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): coerced_value = coerce_types(original_value) assert coerced_value == coerced_expected - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.authenticate_and_create_graph') + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') + @mock.patch('cms.djangoapps.coursegraph.tasks.authenticate_and_create_graph') def test_dump_to_neo4j(self, mock_graph_constructor, mock_matcher_class): """ Tests the dump_to_neo4j method works against a mock @@ -423,8 +423,8 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): assert len(mock_graph.nodes) == 11 self.assertCountEqual(submitted, self.course_strings) - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.authenticate_and_create_graph') + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') + @mock.patch('cms.djangoapps.coursegraph.tasks.authenticate_and_create_graph') def test_dump_to_neo4j_rollback(self, mock_graph_constructor, mock_matcher_class): """ Tests that the the dump_to_neo4j method handles the case where there's @@ -447,8 +447,8 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): self.assertCountEqual(submitted, self.course_strings) - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.authenticate_and_create_graph') + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') + @mock.patch('cms.djangoapps.coursegraph.tasks.authenticate_and_create_graph') @ddt.data((True, 2), (False, 0)) @ddt.unpack def test_dump_to_neo4j_cache( @@ -480,8 +480,8 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): ) assert len(submitted) == expected_number_courses - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.authenticate_and_create_graph') + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') + @mock.patch('cms.djangoapps.coursegraph.tasks.authenticate_and_create_graph') def test_dump_to_neo4j_published(self, mock_graph_constructor, mock_matcher_class): """ Tests that we only dump those courses that have been published after @@ -506,8 +506,8 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): assert len(submitted) == 1 assert submitted[0] == str(self.course.id) - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.get_course_last_published') - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.get_command_last_run') + @mock.patch('cms.djangoapps.coursegraph.tasks.get_course_last_published') + @mock.patch('cms.djangoapps.coursegraph.tasks.get_command_last_run') @ddt.data( ( str(datetime(2016, 3, 30)), str(datetime(2016, 3, 31)), diff --git a/openedx/core/djangoapps/coursegraph/management/commands/tests/utils.py b/cms/djangoapps/coursegraph/management/commands/tests/utils.py similarity index 100% rename from openedx/core/djangoapps/coursegraph/management/commands/tests/utils.py rename to cms/djangoapps/coursegraph/management/commands/tests/utils.py diff --git a/openedx/core/djangoapps/coursegraph/tasks.py b/cms/djangoapps/coursegraph/tasks.py similarity index 100% rename from openedx/core/djangoapps/coursegraph/tasks.py rename to cms/djangoapps/coursegraph/tasks.py diff --git a/cms/envs/common.py b/cms/envs/common.py index 2cad881cec..31e73c2559 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -1611,7 +1611,7 @@ INSTALLED_APPS = [ 'openedx.core.djangoapps.self_paced', # Coursegraph - 'openedx.core.djangoapps.coursegraph.apps.CoursegraphConfig', + 'cms.djangoapps.coursegraph.apps.CoursegraphConfig', # Credit courses 'openedx.core.djangoapps.credit.apps.CreditConfig', diff --git a/cms/envs/production.py b/cms/envs/production.py index 8b29e3d3f3..82244acbde 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -603,7 +603,7 @@ EXPLICIT_QUEUES = { 'queue': POLICY_CHANGE_GRADES_ROUTING_KEY}, 'cms.djangoapps.contentstore.tasks.update_search_index': { 'queue': UPDATE_SEARCH_INDEX_JOB_QUEUE}, - 'openedx.core.djangoapps.coursegraph.tasks.dump_course_to_neo4j': { + 'cms.djangoapps.coursegraph.tasks.dump_course_to_neo4j': { 'queue': COURSEGRAPH_JOB_QUEUE}, } diff --git a/lms/envs/common.py b/lms/envs/common.py index 83709b11db..bf81a9769e 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -3083,10 +3083,6 @@ INSTALLED_APPS = [ 'openedx.core.djangoapps.content.block_structure.apps.BlockStructureConfig', 'lms.djangoapps.course_blocks', - - # Coursegraph - 'openedx.core.djangoapps.coursegraph.apps.CoursegraphConfig', - # Mailchimp Syncing 'lms.djangoapps.mailing', From c7693cfa2053f0285ff531a43e8837abc0879669 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 20 Jan 2022 18:30:41 -0500 Subject: [PATCH 38/52] docs: enrich dump_to_neo4j help text & docstrings Move most docs out of docstring and into programatically- displayable argument help text. Also, the 'Example Usage' was out of date. This commit updates it to: * use `./manage.py cms ...' instead of `./manage.py lms ...', and * use `--port` instead of `--https_port`. --- .../management/commands/dump_to_neo4j.py | 80 +++++++++++++------ 1 file changed, 54 insertions(+), 26 deletions(-) diff --git a/cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py b/cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py index 5fcaefafcb..6da97b4a22 100644 --- a/cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py +++ b/cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py @@ -1,6 +1,17 @@ """ This file contains a management command for exporting the modulestore to -neo4j, a graph database. +Neo4j, a graph database. + +Example usages: + + # Dump all courses published since last dump. + python manage.py cms dump_to_neo4j --host localhost --port 7473 \ + --secure --user user --password password + + # Specify certain courses instead of dumping all of them. + python manage.py cms dump_to_neo4j --host localhost --port 7473 \ + --secure --user user --password password \ + --courses 'course-v1:A+B+1' 'course-v1:A+B+2' """ @@ -16,38 +27,55 @@ log = logging.getLogger(__name__) class Command(BaseCommand): """ - Command to dump modulestore data to neo4j - - Takes the following named arguments: - host: the host of the neo4j server - port: the port on the neo4j server that accepts Bolt requests - secure: if set, connects to server over Bolt/TLS, otherwise uses Bolt - user: the username for the neo4j user - password: the user's password - courses: list of course key strings to serialize. If not specified, all - courses in the modulestore are serialized. - override: if true, dump all--or all specified--courses, regardless of when - they were last dumped. If false, or not set, only dump those courses that - were updated since the last time the command was run. - - Example usage: - python manage.py lms dump_to_neo4j --host localhost --https_port 7473 \ - --secure --user user --password password --settings=production + Dump recently-published course(s) over to a CourseGraph (Neo4j) instance. """ help = dedent(__doc__).strip() def add_arguments(self, parser): - parser.add_argument('--host', type=str) - parser.add_argument('--port', type=int, default=7687) - parser.add_argument('--secure', action='store_true') - parser.add_argument('--user', type=str) - parser.add_argument('--password', type=str) - parser.add_argument('--courses', type=str, nargs='*') - parser.add_argument('--skip', type=str, nargs='*') + parser.add_argument( + '--host', + type=str, + help="the hostname of the Neo4j server", + ) + parser.add_argument( + '--port', + type=int, + default=7687, + help="the port on the Neo4j server that accepts Bolt requests", + ) + parser.add_argument( + '--secure', + action='store_true', + help="connect to server over Bolt/TLS instead of plain unencrypted Bolt", + ) + parser.add_argument( + '--user', + type=str, + help="the username of the Neo4j user", + ) + parser.add_argument( + '--password', + type=str, + help="the password of the Neo4j user", + ) + parser.add_argument( + '--courses', + metavar='KEY', + type=str, + nargs='*', + help="keys of courses to serialize; if omitted all courses in system are serialized", + ) + parser.add_argument( + '--skip', + metavar='KEY', + type=str, + nargs='*', + help="keys of courses to NOT to serialize", + ) parser.add_argument( '--override', action='store_true', - help='dump all--or all specified--courses, ignoring cache', + help="dump all courses regardless of when they were last published", ) def handle(self, *args, **options): From 1bf8af5f721bb0ccfb6f8882b3e24d2521e62b25 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 20 Jan 2022 18:41:34 -0500 Subject: [PATCH 39/52] feat: specify dump_to_neo4j defaults in COURSEGRAPH_CONNECTION Introduce a new CMS settings COURSEGRAPH_CONNECTION, which allows operators to specify default connection paramters for a Neo4j instance. This has three purposes: * The `./manage.py cms dump_to_neo4j` management command will be much easier for developers and operators to type out because connection arguments can now be omitted. Note that connection arguments, if supplied, will override the arguments specified in CMS settings. * The automatic push-to-coursegraph-on-publish-signal introduced in subsequent commits can use these connection settings. * The CourseGraph Django admin actions introduced in subsequent commits can use these connection settings. --- .../management/commands/dump_to_neo4j.py | 19 ++++-- .../commands/tests/test_dump_to_neo4j.py | 40 ++++++++++++- cms/djangoapps/coursegraph/tasks.py | 59 +++++++++---------- cms/envs/common.py | 18 +++++- cms/envs/devstack.py | 11 ++++ 5 files changed, 109 insertions(+), 38 deletions(-) diff --git a/cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py b/cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py index 6da97b4a22..40afe7ffbe 100644 --- a/cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py +++ b/cms/djangoapps/coursegraph/management/commands/dump_to_neo4j.py @@ -5,13 +5,17 @@ Neo4j, a graph database. Example usages: # Dump all courses published since last dump. + # Use connection parameters from `settings.COURSEGRAPH_SETTINGS`. + python manage.py cms dump_to_neo4j + + # Dump all courses published since last dump. + # Use custom connection parameters. python manage.py cms dump_to_neo4j --host localhost --port 7473 \ --secure --user user --password password # Specify certain courses instead of dumping all of them. - python manage.py cms dump_to_neo4j --host localhost --port 7473 \ - --secure --user user --password password \ - --courses 'course-v1:A+B+1' 'course-v1:A+B+2' + # Use connection parameters from `settings.COURSEGRAPH_SETTINGS`. + python manage.py cms dump_to_neo4j --courses 'course-v1:A+B+1' 'course-v1:A+B+2' """ @@ -40,7 +44,6 @@ class Command(BaseCommand): parser.add_argument( '--port', type=int, - default=7687, help="the port on the Neo4j server that accepts Bolt requests", ) parser.add_argument( @@ -85,9 +88,13 @@ class Command(BaseCommand): """ mss = ModuleStoreSerializer.create(options['courses'], options['skip']) - + connection_overrides = { + key: options[key] + for key in ["host", "port", "secure", "user", "password"] + } submitted_courses, skipped_courses = mss.dump_courses_to_neo4j( - options, override_cache=options['override'] + connection_overrides=connection_overrides, + override_cache=options['override'], ) log.info( diff --git a/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py b/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py index 37d2dfaf66..dd07e0f08a 100644 --- a/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py +++ b/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py @@ -8,12 +8,13 @@ from datetime import datetime from unittest import mock import ddt from django.core.management import call_command +from django.test.utils import override_settings from edx_toggles.toggles.testutils import override_waffle_switch from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory import openedx.core.djangoapps.content.block_structure.config as block_structure_config -from cms.djangoapps.content.block_structure.signals import update_block_structure_on_course_publish +from openedx.core.djangoapps.content.block_structure.signals import update_block_structure_on_course_publish from cms.djangoapps.coursegraph.management.commands.dump_to_neo4j import ModuleStoreSerializer from cms.djangoapps.coursegraph.management.commands.tests.utils import MockGraph, MockNodeMatcher from cms.djangoapps.coursegraph.tasks import ( @@ -219,6 +220,43 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): number_rollbacks=0, ) + @mock.patch('cms.djangoapps.coursegraph.tasks.Graph', autospec=True) + @override_settings( + COURSEGRAPH_CONNECTION=dict( + protocol='bolt', + host='coursegraph.example.edu', + port=7777, + secure=True, + user="neo4j", + password="default-password", + ) + ) + def test_dump_to_neo4j_connection_defaults(self, mock_graph_class): + """ + Test that user can override individual settings.COURSEGRAPH_CONNECTION parameters + by passing them to `dump_to_neo4j`, whilst falling back to the ones that they + don't override. + """ + call_command( + 'dump_to_neo4j', + courses=self.course_strings[:1], + port=7788, + secure=False, + password="overridden-password", + ) + mock_graph_class.assert_called_once_with( + + # From settings: + protocol='bolt', + host='coursegraph.example.edu', + user="neo4j", + + # Overriden by command: + port=7788, + secure=False, + password="overridden-password", + ) + class SomeThing: """Just to test the stringification of an object.""" diff --git a/cms/djangoapps/coursegraph/tasks.py b/cms/djangoapps/coursegraph/tasks.py index ec943e7527..2dc359974c 100644 --- a/cms/djangoapps/coursegraph/tasks.py +++ b/cms/djangoapps/coursegraph/tasks.py @@ -7,6 +7,7 @@ neo4j, a graph database. import logging from celery import shared_task +from django.conf import settings from django.utils import timezone from edx_django_utils.cache import RequestCache from edx_django_utils.monitoring import set_code_owner_attribute @@ -268,14 +269,14 @@ def should_dump_course(course_key, graph): @shared_task @set_code_owner_attribute -def dump_course_to_neo4j(course_key_string, credentials): +def dump_course_to_neo4j(course_key_string, connection_overrides=None): """ Serializes a course and writes it to neo4j. Arguments: - course_key: course key for the course to be exported - credentials (dict): the necessary credentials to connect - to neo4j and create a py2neo `Graph` obje + course_key_string: course key for the course to be exported + connection_overrides (dict): overrides to Neo4j connection + parameters specified in `settings.COURSEGRAPH_CONNECTION`. """ course_key = CourseKey.from_string(course_key_string) nodes, relationships = serialize_course(course_key) @@ -286,7 +287,9 @@ def dump_course_to_neo4j(course_key_string, credentials): len(relationships), ) - graph = authenticate_and_create_graph(credentials) + graph = authenticate_and_create_graph( + connection_overrides=connection_overrides + ) transaction = graph.begin() course_string = str(course_key) @@ -346,13 +349,13 @@ class ModuleStoreSerializer: course_keys = [course_key for course_key in course_keys if course_key not in skip_keys] return cls(course_keys) - def dump_courses_to_neo4j(self, credentials, override_cache=False): + def dump_courses_to_neo4j(self, connection_overrides=None, override_cache=False): """ Method that iterates through a list of courses in a modulestore, serializes them, then submits tasks to write them to neo4j. Arguments: - credentials (dict): the necessary credentials to connect - to neo4j and create a py2neo `Graph` object + connection_overrides (dict): overrides to Neo4j connection + parameters specified in `settings.COURSEGRAPH_CONNECTION`. override_cache: serialize the courses even if they'be been recently serialized @@ -365,7 +368,7 @@ class ModuleStoreSerializer: submitted_courses = [] skipped_courses = [] - graph = authenticate_and_create_graph(credentials) + graph = authenticate_and_create_graph(connection_overrides) for index, course_key in enumerate(self.course_keys): # first, clear the request cache to prevent memory leaks @@ -389,36 +392,32 @@ class ModuleStoreSerializer: ) dump_course_to_neo4j.apply_async( - args=[str(course_key), credentials], + kwargs=dict( + course_key_string=str(course_key), + connection_overrides=connection_overrides, + ) ) submitted_courses.append(str(course_key)) return submitted_courses, skipped_courses -def authenticate_and_create_graph(credentials): +def authenticate_and_create_graph(connection_overrides=None): """ This function authenticates with neo4j and creates a py2neo graph object + Arguments: - credentials (dict): a dictionary of credentials used to authenticate, - and then create, a py2neo graph object. + connection_overrides (dict): overrides to Neo4j connection + parameters specified in `settings.COURSEGRAPH_CONNECTION`. Returns: a py2neo `Graph` object. """ - - host = credentials['host'] - port = credentials['port'] - secure = credentials['secure'] - neo4j_user = credentials['user'] - neo4j_password = credentials['password'] - - graph = Graph( - protocol='bolt', - password=neo4j_password, - user=neo4j_user, - address=host, - port=port, - secure=secure, - ) - - return graph + provided_overrides = { + key: value + for key, value in (connection_overrides or {}).items() + # Drop overrides whose values are `None`. Note that `False` is a + # legitimate override value that we don't want to drop here. + if value is not None + } + connection_with_overrides = {**settings.COURSEGRAPH_CONNECTION, **provided_overrides} + return Graph(**connection_with_overrides) diff --git a/cms/envs/common.py b/cms/envs/common.py index 31e73c2559..91cbbbc215 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -2235,7 +2235,23 @@ POLICY_CHANGE_TASK_RATE_LIMIT = '300/h' # .. setting_default: value of LOW_PRIORITY_QUEUE # .. setting_description: The name of the Celery queue to which CourseGraph refresh # tasks will be sent -COURSEGRAPH_JOB_QUEUE = LOW_PRIORITY_QUEUE +COURSEGRAPH_JOB_QUEUE: str = LOW_PRIORITY_QUEUE + +# .. setting_name: COURSEGRAPH_CONNECTION +# .. setting_default: 'bolt+s://localhost:7687', in dictionary form. +# .. setting_description: Dictionary specifying Neo4j connection parameters for +# CourseGraph refresh. Accepted keys are protocol ('bolt' or 'http'), +# secure (bool), host (str), port (int), user (str), and password (str). +# See https://py2neo.org/2021.1/profiles.html#individual-settings for a +# a description of each of those keys. +COURSEGRAPH_CONNECTION: dict = { + "protocol": "bolt", + "secure": True, + "host": "localhost", + "port": 7687, + "user": "neo4j", + "password": None, +} ########## Settings for video transcript migration tasks ############ VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE = DEFAULT_PRIORITY_QUEUE diff --git a/cms/envs/devstack.py b/cms/envs/devstack.py index 6b632ddaf1..6c67d4d389 100644 --- a/cms/envs/devstack.py +++ b/cms/envs/devstack.py @@ -256,6 +256,17 @@ FEATURES['ENABLE_PREREQUISITE_COURSES'] = True # (ref MST-637) PROCTORING_USER_OBFUSCATION_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' +############## CourseGraph devstack settings ############################ + +COURSEGRAPH_CONNECTION: dict = { + "protocol": "bolt", + "secure": False, + "host": "edx.devstack.coursegraph", + "port": 7687, + "user": "neo4j", + "password": "edx", +} + #################### Webpack Configuration Settings ############################## WEBPACK_LOADER['DEFAULT']['TIMEOUT'] = 5 From 696984a2bdb08a6a4a7069115f7c0ffed620ed97 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Mon, 24 Jan 2022 11:51:56 -0500 Subject: [PATCH 40/52] feat: dump to cousegraph on course publish signal Previously, CourseGraph needed to be kept up-to-date by running `./manage.py dump_to_neo4j ...` manually or on a cron timer. This introduces a CMS new setting: COURSEGRAPH_DUMP_COURSE_ON_PUBLISH. When enabled, the CMS course_published signal handler will asynchronously dump each individual course to CourseGraph when it is published. This follows a pattern established by other subsystems like learning_sequences and special exam registration, both of which fire off asynchronous post-processing tasks from the course- publish handler. --- cms/djangoapps/contentstore/signals/handlers.py | 8 ++++++++ cms/envs/common.py | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/cms/djangoapps/contentstore/signals/handlers.py b/cms/djangoapps/contentstore/signals/handlers.py index a67f958381..9945277a13 100644 --- a/cms/djangoapps/contentstore/signals/handlers.py +++ b/cms/djangoapps/contentstore/signals/handlers.py @@ -5,6 +5,7 @@ import logging from datetime import datetime from functools import wraps +from django.conf import settings from django.core.cache import cache from django.dispatch import receiver from pytz import UTC @@ -55,6 +56,9 @@ def listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable= update_search_index, update_special_exams_and_publish ) + from cms.djangoapps.coursegraph.tasks import ( + dump_course_to_neo4j + ) # register special exams asynchronously course_key_str = str(course_key) @@ -64,6 +68,10 @@ def listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable= # Push the course outline to learning_sequences asynchronously. update_outline_from_modulestore_task.delay(course_key_str) + if settings.COURSEGRAPH_DUMP_COURSE_ON_PUBLISH: + # Push the course out to CourseGraph asynchronously. + dump_course_to_neo4j.delay(course_key_str) + # Finally, call into the course search subsystem # to kick off an indexing action if CoursewareSearchIndexer.indexing_is_enabled() and CourseAboutSearchIndexer.indexing_is_enabled(): diff --git a/cms/envs/common.py b/cms/envs/common.py index 91cbbbc215..759f474d68 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -2253,6 +2253,16 @@ COURSEGRAPH_CONNECTION: dict = { "password": None, } +# .. toggle_name: COURSEGRAPH_DUMP_COURSE_ON_PUBLISH +# .. toggle_implementation: DjangoSetting +# .. toggle_creation_date: 2022-01-27 +# .. toggle_use_cases: open_edx +# .. toggle_default: False +# .. toggle_description: Whether, upon publish, a course should automatically +# be exported to Neo4j via the connection parameters specified in +# `COURSEGRAPH_CONNECTION`. +COURSEGRAPH_DUMP_COURSE_ON_PUBLISH: bool = False + ########## Settings for video transcript migration tasks ############ VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE = DEFAULT_PRIORITY_QUEUE From d75a32c00935259450491614e46868bbe80a80d0 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Mon, 7 Feb 2022 16:13:53 -0500 Subject: [PATCH 41/52] refactor: read course publish date from overview, not block structure The `get_course_last_published` function is used by CourseGraph to determine whether or not a course should be dumped to Neo4j. If the course hasn't been published since it was last dumped to Neo4j, then it can be skipped (unless the override_cache option is enabled). The function was previously built using the BlockStructure data model. While this worked fine in Production instances that enable `block_structure.storage_backing_for_cache`, this implementation did NOT work in development environments, which do not use the BlockStrcture model. Instead, we switch to using CourseOverview.modified to approximate when a course was last published. This is method has fewer moving parts and is universally available across instances. --- .../commands/tests/test_dump_to_neo4j.py | 9 ++++-- cms/djangoapps/coursegraph/tasks.py | 31 +++++++++---------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py b/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py index dd07e0f08a..32fef0a887 100644 --- a/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py +++ b/cms/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py @@ -220,6 +220,7 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): number_rollbacks=0, ) + @mock.patch('cms.djangoapps.coursegraph.tasks.NodeMatcher') @mock.patch('cms.djangoapps.coursegraph.tasks.Graph', autospec=True) @override_settings( COURSEGRAPH_CONNECTION=dict( @@ -231,12 +232,15 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): password="default-password", ) ) - def test_dump_to_neo4j_connection_defaults(self, mock_graph_class): + def test_dump_to_neo4j_connection_defaults(self, mock_graph_class, mock_matcher_class): """ Test that user can override individual settings.COURSEGRAPH_CONNECTION parameters by passing them to `dump_to_neo4j`, whilst falling back to the ones that they don't override. """ + self.setup_mock_graph( + mock_matcher_class, mock_graph_class + ) call_command( 'dump_to_neo4j', courses=self.course_strings[:1], @@ -244,7 +248,8 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): secure=False, password="overridden-password", ) - mock_graph_class.assert_called_once_with( + assert mock_graph_class.call_args.args == () + assert mock_graph_class.call_args.kwargs == dict( # From settings: protocol='bolt', diff --git a/cms/djangoapps/coursegraph/tasks.py b/cms/djangoapps/coursegraph/tasks.py index 2dc359974c..e2d4bf5b09 100644 --- a/cms/djangoapps/coursegraph/tasks.py +++ b/cms/djangoapps/coursegraph/tasks.py @@ -134,29 +134,26 @@ def get_command_last_run(course_key, graph): def get_course_last_published(course_key): """ - We use the CourseStructure table to get when this course was last - published. + Approximately when was a course last published? + + We use the 'modified' column in the CourseOverview table as a quick and easy + (although perhaps inexact) way of determining when a course was last + published. This works because CourseOverview rows are re-written upon + course publish. + Args: course_key: a CourseKey - Returns: The datetime the course was last published at, converted into - text, or None, if there's no record of the last time this course - was published. + Returns: The datetime the course was last published at, stringified. + Uses Python's default str(...) implementation for datetimes, which + is sortable and similar to ISO 8601: + https://docs.python.org/3/library/datetime.html#datetime.date.__str__ """ # Import is placed here to avoid model import at project startup. - from xmodule.modulestore.django import modulestore - from openedx.core.djangoapps.content.block_structure.models import BlockStructureModel - from openedx.core.djangoapps.content.block_structure.exceptions import BlockStructureNotFound + from openedx.core.djangoapps.content.course_overviews.models import CourseOverview - store = modulestore() - course_usage_key = store.make_course_usage_key(course_key) - try: - structure = BlockStructureModel.get(course_usage_key) - course_last_published_date = str(structure.modified) - except BlockStructureNotFound: - course_last_published_date = None - - return course_last_published_date + approx_last_published = CourseOverview.get_from_id(course_key).modified + return str(approx_last_published) def strip_branch_and_version(location): From d16fe9d4271055380b0efce3122c5d4cd208517e Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Mon, 24 Jan 2022 11:52:49 -0500 Subject: [PATCH 42/52] feat: add admin action for dump to coursegraph This introduces two admin actions: * Dump to CourseGraph (respect cache), and * Dump to CourseGraph (override cache) which allow admins to select a collection of courses from Django admin and dump them to the Neo4j instance specified by settings.COURSEGRAPH_CONNECTION, with or without respecting the cache (that is: whether the course has already been dumped since its last publishing). --- cms/djangoapps/coursegraph/admin.py | 123 ++++++++++ cms/djangoapps/coursegraph/models.py | 21 ++ cms/djangoapps/coursegraph/tests/__init__.py | 0 .../coursegraph/tests/test_admin.py | 227 ++++++++++++++++++ 4 files changed, 371 insertions(+) create mode 100644 cms/djangoapps/coursegraph/admin.py create mode 100644 cms/djangoapps/coursegraph/models.py create mode 100644 cms/djangoapps/coursegraph/tests/__init__.py create mode 100644 cms/djangoapps/coursegraph/tests/test_admin.py diff --git a/cms/djangoapps/coursegraph/admin.py b/cms/djangoapps/coursegraph/admin.py new file mode 100644 index 0000000000..f79fa909d2 --- /dev/null +++ b/cms/djangoapps/coursegraph/admin.py @@ -0,0 +1,123 @@ +""" +Admin site bindings for coursegraph +""" +import logging + +from django.contrib import admin, messages +from django.utils.translation import gettext as _ +from edx_django_utils.admin.mixins import ReadOnlyAdminMixin + +from .models import CourseGraphCourseDump +from .tasks import ModuleStoreSerializer + +log = logging.getLogger(__name__) + + +@admin.action( + permissions=['change'], + description=_("Dump courses to CourseGraph (respect cache)"), +) +def dump_courses(modeladmin, request, queryset): + """ + Admin action to enqueue Dump-to-CourseGraph tasks for a set of courses, + excluding courses that haven't been published since they were last dumped. + + queryset is a QuerySet of CourseGraphCourseDump objects, which are just + CourseOverview objects under the hood. + """ + all_course_keys = queryset.values_list('id', flat=True) + serializer = ModuleStoreSerializer(all_course_keys) + try: + submitted, skipped = serializer.dump_courses_to_neo4j() + # Unfortunately there is no unified base class for the reasonable + # exceptions we could expect from py2neo (connection unavailable, bolt protocol + # error, and so on), so we just catch broadly, show a generic error banner, + # and then log the exception for site operators to look at. + except Exception as err: # pylint: disable=broad-except + log.exception( + "Failed to enqueue CourseGraph dumps to Neo4j (respecting cache): %s", + ", ".join(str(course_key) for course_key in all_course_keys), + ) + modeladmin.message_user( + request, + _("Error enqueueing dumps for {} course(s): {}").format( + len(all_course_keys), str(err) + ), + level=messages.ERROR, + ) + return + if submitted: + modeladmin.message_user( + request, + _( + "Enqueued dumps for {} course(s). Skipped {} unchanged course(s)." + ).format(len(submitted), len(skipped)), + level=messages.SUCCESS, + ) + else: + modeladmin.message_user( + request, + _( + "Skipped all {} course(s), as they were unchanged.", + ).format(len(skipped)), + level=messages.WARNING, + ) + + +@admin.action( + permissions=['change'], + description=_("Dump courses to CourseGraph (override cache)") +) +def dump_courses_overriding_cache(modeladmin, request, queryset): + """ + Admin action to enqueue Dump-to-CourseGraph tasks for a set of courses + (whether or not they have been published recently). + + queryset is a QuerySet of CourseGraphCourseDump objects, which are just + CourseOverview objects under the hood. + """ + all_course_keys = queryset.values_list('id', flat=True) + serializer = ModuleStoreSerializer(all_course_keys) + try: + submitted, _skipped = serializer.dump_courses_to_neo4j(override_cache=True) + # Unfortunately there is no unified base class for the reasonable + # exceptions we could expect from py2neo (connection unavailable, bolt protocol + # error, and so on), so we just catch broadly, show a generic error banner, + # and then log the exception for site operators to look at. + except Exception as err: # pylint: disable=broad-except + log.exception( + "Failed to enqueue CourseGraph Neo4j course dumps (overriding cache): %s", + ", ".join(str(course_key) for course_key in all_course_keys), + ) + modeladmin.message_user( + request, + _("Error enqueueing dumps for {} course(s): {}").format( + len(all_course_keys), str(err) + ), + level=messages.ERROR, + ) + return + modeladmin.message_user( + request, + _("Enqueued dumps for {} course(s).").format(len(submitted)), + level=messages.SUCCESS, + ) + + +@admin.register(CourseGraphCourseDump) +class CourseGraphCourseDumpAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): + """ + Model admin for "Course graph course dumps". + + Just a read-only table with some useful metadata, allowing admin users to + select courses to be dumped to CourseGraph. + """ + list_display = [ + 'id', + 'display_name', + 'modified', + 'enrollment_start', + 'enrollment_end', + ] + search_fields = ['id', 'display_name'] + actions = [dump_courses, dump_courses_overriding_cache] diff --git a/cms/djangoapps/coursegraph/models.py b/cms/djangoapps/coursegraph/models.py new file mode 100644 index 0000000000..f053dc9993 --- /dev/null +++ b/cms/djangoapps/coursegraph/models.py @@ -0,0 +1,21 @@ +""" +(Proxy) models supporting CourseGraph. +""" + +from openedx.core.djangoapps.content.course_overviews.models import CourseOverview + + +class CourseGraphCourseDump(CourseOverview): + """ + Proxy model for CourseOverview. + + Does *not* create/update/delete CourseOverview objects - only reads the objects. + Uses the course IDs of the CourseOverview objects to determine which courses + can be dumped to CourseGraph. + """ + class Meta: + proxy = True + + def __str__(self): + """Represent ourselves with the course key.""" + return str(self.id) diff --git a/cms/djangoapps/coursegraph/tests/__init__.py b/cms/djangoapps/coursegraph/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cms/djangoapps/coursegraph/tests/test_admin.py b/cms/djangoapps/coursegraph/tests/test_admin.py new file mode 100644 index 0000000000..21a26d8450 --- /dev/null +++ b/cms/djangoapps/coursegraph/tests/test_admin.py @@ -0,0 +1,227 @@ +""" +Shallow tests for CourseGraph dump-queueing Django admin interface. + +See ..management.commands.tests.test_dump_to_neo4j for more comprehensive +tests of dump_course_to_neo4j. +""" + +from unittest import mock + +import py2neo +from django.test import TestCase +from django.test.utils import override_settings +from freezegun import freeze_time + +from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory +from openedx.core.djangoapps.content.course_overviews.models import CourseOverview + +from .. import admin, tasks + + +_coursegraph_connection = { + "protocol": "bolt", + "secure": True, + "host": "example.edu", + "port": 7687, + "user": "neo4j", + "password": "fake-coursegraph-password", +} + +_configure_coursegraph_connection = override_settings( + COURSEGRAPH_CONNECTION=_coursegraph_connection, +) + +_patch_log_exception = mock.patch.object( + admin.log, 'exception', autospec=True +) + +_patch_apply_dump_task = mock.patch.object( + tasks.dump_course_to_neo4j, 'apply_async' +) + +_pretend_last_course_dump_was_may_2020 = mock.patch.object( + tasks, + 'get_command_last_run', + new=(lambda _key, _graph: "2020-05-01"), +) + +_patch_neo4j_graph = mock.patch.object( + tasks, 'Graph', autospec=True +) + +_make_neo4j_graph_raise = mock.patch.object( + tasks, 'Graph', side_effect=py2neo.ConnectionUnavailable( + 'we failed to connect or something!' + ) +) + + +class CourseGraphAdminActionsTestCase(TestCase): + """ + Test CourseGraph Django admin actions. + """ + + @classmethod + def setUpTestData(cls): + """ + Make course overviews with varying modification dates. + """ + super().setUpTestData() + cls.course_updated_in_april = CourseOverviewFactory(run='april_update') + cls.course_updated_in_june = CourseOverviewFactory(run='june_update') + cls.course_updated_in_july = CourseOverviewFactory(run='july_update') + cls.course_updated_in_august = CourseOverviewFactory(run='august_update') + + # For each course overview, make an arbitrary update and then save() + # so that its `.modified` date is set. + with freeze_time("2020-04-01"): + cls.course_updated_in_april.marketing_url = "https://example.org" + cls.course_updated_in_april.save() + with freeze_time("2020-06-01"): + cls.course_updated_in_june.marketing_url = "https://example.org" + cls.course_updated_in_june.save() + with freeze_time("2020-07-01"): + cls.course_updated_in_july.marketing_url = "https://example.org" + cls.course_updated_in_july.save() + with freeze_time("2020-08-01"): + cls.course_updated_in_august.marketing_url = "https://example.org" + cls.course_updated_in_august.save() + + @_configure_coursegraph_connection + @_pretend_last_course_dump_was_may_2020 + @_patch_neo4j_graph + @_patch_apply_dump_task + @_patch_log_exception + def test_dump_courses(self, mock_log_exception, mock_apply_dump_task, mock_neo4j_graph): + """ + Test that dump_courses admin action dumps requested courses iff they have + been modified since the last dump to coursegraph. + """ + modeladmin_mock = mock.MagicMock() + + # Request all courses except the August-updated one + requested_course_keys = { + str(self.course_updated_in_april.id), + str(self.course_updated_in_june.id), + str(self.course_updated_in_july.id), + } + admin.dump_courses( + modeladmin=modeladmin_mock, + request=mock.MagicMock(), + queryset=CourseOverview.objects.filter(id__in=requested_course_keys), + ) + + # User should have been messaged + assert modeladmin_mock.message_user.call_count == 1 + assert modeladmin_mock.message_user.call_args.args[1] == ( + "Enqueued dumps for 2 course(s). Skipped 1 unchanged course(s)." + ) + + # For enqueueing, graph should've been authenticated once, using configured settings. + assert mock_neo4j_graph.call_count == 1 + assert mock_neo4j_graph.call_args.args == () + assert mock_neo4j_graph.call_args.kwargs == _coursegraph_connection + + # No errors should've been logged. + assert mock_log_exception.call_count == 0 + + # April course should have been skipped because the command was last run in May. + # Dumps for June and July courses should have been enqueued. + assert mock_apply_dump_task.call_count == 2 + actual_dumped_course_keys = { + call_args.kwargs['kwargs']['course_key_string'] + for call_args in mock_apply_dump_task.call_args_list + } + expected_dumped_course_keys = { + str(self.course_updated_in_june.id), + str(self.course_updated_in_july.id), + } + assert actual_dumped_course_keys == expected_dumped_course_keys + + @_configure_coursegraph_connection + @_pretend_last_course_dump_was_may_2020 + @_patch_neo4j_graph + @_patch_apply_dump_task + @_patch_log_exception + def test_dump_courses_overriding_cache(self, mock_log_exception, mock_apply_dump_task, mock_neo4j_graph): + """ + Test that dump_coursese_overriding_cach admin action dumps requested courses + whether or not they been modified since the last dump to coursegraph. + """ + modeladmin_mock = mock.MagicMock() + + # Request all courses except the August-updated one + requested_course_keys = { + str(self.course_updated_in_april.id), + str(self.course_updated_in_june.id), + str(self.course_updated_in_july.id), + } + admin.dump_courses_overriding_cache( + modeladmin=modeladmin_mock, + request=mock.MagicMock(), + queryset=CourseOverview.objects.filter(id__in=requested_course_keys), + ) + + # User should have been messaged + assert modeladmin_mock.message_user.call_count == 1 + assert modeladmin_mock.message_user.call_args.args[1] == ( + "Enqueued dumps for 3 course(s)." + ) + + # For enqueueing, graph should've been authenticated once, using configured settings. + assert mock_neo4j_graph.call_count == 1 + assert mock_neo4j_graph.call_args.args == () + assert mock_neo4j_graph.call_args.kwargs == _coursegraph_connection + + # No errors should've been logged. + assert mock_log_exception.call_count == 0 + + # April, June, and July courses should have all been dumped. + assert mock_apply_dump_task.call_count == 3 + actual_dumped_course_keys = { + call_args.kwargs['kwargs']['course_key_string'] + for call_args in mock_apply_dump_task.call_args_list + } + expected_dumped_course_keys = { + str(self.course_updated_in_april.id), + str(self.course_updated_in_june.id), + str(self.course_updated_in_july.id), + } + assert actual_dumped_course_keys == expected_dumped_course_keys + + @_configure_coursegraph_connection + @_pretend_last_course_dump_was_may_2020 + @_make_neo4j_graph_raise + @_patch_apply_dump_task + @_patch_log_exception + def test_dump_courses_error(self, mock_log_exception, mock_apply_dump_task, mock_neo4j_graph): + """ + Test that the dump_courses admin action dumps messages the user if an error + occurs when trying to enqueue course dumps. + """ + modeladmin_mock = mock.MagicMock() + + # Request dump of all four courses. + admin.dump_courses( + modeladmin=modeladmin_mock, + request=mock.MagicMock(), + queryset=CourseOverview.objects.all() + ) + + # Admin user should have been messaged about failure. + assert modeladmin_mock.message_user.call_count == 1 + assert modeladmin_mock.message_user.call_args.args[1] == ( + "Error enqueueing dumps for 4 course(s): we failed to connect or something!" + ) + + # For enqueueing, graph should've been authenticated once, using configured settings. + assert mock_neo4j_graph.call_count == 1 + assert mock_neo4j_graph.call_args.args == () + assert mock_neo4j_graph.call_args.kwargs == _coursegraph_connection + + # Exception should have been logged. + assert mock_log_exception.call_count == 1 + assert "Failed to enqueue" in mock_log_exception.call_args.args[0] + + # No courses should have been dumped. + assert mock_apply_dump_task.call_count == 0 From 42fcfc8217eb83ca77060ed901745b3808a697c9 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Mon, 28 Mar 2022 14:45:00 -0400 Subject: [PATCH 43/52] docs: update CourseGraph README, notably w.r.t. new Tutor plugin Update the README of the CMS's CourseGraph support app: * Point to the newly-developed CourseGraph plugin for Tutor, and remove some prose that's now redundant with the Tutor plugin's README. * Add a link to the now-public CourseGraph Queries wiki page. * Capitalize the G in CourseGraph. * Fix a couple misc. formatting things. --- cms/djangoapps/coursegraph/README.rst | 49 +++++++++++++++++++-------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/cms/djangoapps/coursegraph/README.rst b/cms/djangoapps/coursegraph/README.rst index 0f9089ea42..18f0f60bdd 100644 --- a/cms/djangoapps/coursegraph/README.rst +++ b/cms/djangoapps/coursegraph/README.rst @@ -1,37 +1,54 @@ -Coursegraph Support +CourseGraph Support ------------------- -This app exists to write data to "Coursegraph", a tool enabling Open edX developers and support specialists to inspect their platform instance's learning content. Coursegraph itself is simply an instance of Neo4j, which is an open-source graph database with a web interface. +This app exists to write data to "CourseGraph", a tool enabling Open edX developers and support specialists to inspect their platform instance's learning content. CourseGraph itself is simply an instance of `Neo4j`_, which is an open-source graph database with a Web interface. + +.. _Neo4j: https://neo4j.com Deploying Coursegraph ===================== -As of the Maple Open edX release, Coursegraph is *not* automatically provisioned by the community installation, and is *not* considered a "supported" part of the platform. However, operators may find the `neo4j Ansible playbook`_ useful as a starting point for deploying their own Coursegraph instance. Alternatively, Neo4j also maintains an official `Docker image`_. +There are two ways to deploy CourseGraph: -In order for Coursegraph to have queryable data, learning content from LMS must be written to Coursegraph using the ``dump_to_neo4j`` management command included in this app. In order for the data to stay up to date, it must be periodically refreshed, either manually or via an automation server such as Jenkins. +* For operators using Tutor, there is a `CourseGraph plugin for Tutor`_ that is currently released as "Beta". Nutmeg is the earliest Open edX release that the plugin will work alongside. -**Please note**: Access to a populated Coursegraph instance confers access to all the learning content in the related Open edX LMS/CMS. The basic authentication provided by Neo4j may or may not be sufficient for your security needs. Consider taking additional security measures, such as restricting Coursegraph access to only users on a private VPN. +* For operators still using the old Ansible installation pathway, there exists a `neo4j Ansible playbook`_. Be warned that this method is not well-documented nor officially supported. + +In order for CourseGraph to have queryable, up-to-date data, learning content from CMS must be written to CourseGraph regularly. That is where this Django app comes into play. For details on the various ways to write CMS data to CourseGraph, visit the `operations section of the CourseGraph Tutor plugin docs`_. + +**Please note**: Access to a populated CourseGraph instance confers access to all the learning content in the associated Open edX CMS (Studio). The basic authentication provided by Neo4j may or may not be sufficient for your security needs. Consider taking additional security measures, such as restricting CourseGraph access to only users on a private VPN. .. _neo4j Ansible playbook: https://github.com/edx/configuration/blob/master/playbooks/neo4j.yml -.. _Docker image: https://neo4j.com/developer/docker-run-neo4j/ +.. _CourseGraph plugin for Tutor: https://github.com/openedx/tutor-contrib-coursegraph/ +.. _operations section of the CourseGraph Tutor plugin docs: https://github.com/openedx/tutor-contrib-coursegraph/#managing-data -Coursegraph in Devstack -======================= +Running CourseGraph locally +=========================== -Coursegraph is included as an "extra" component in the `Open edX Devstack`_. That is, it is not run or provisioned by default, but can be enabled on-demand. +In some circumstances, you may want to run CourseGraph locally, connected to a development-mode Open edX instance. You can do this in both Tutor and Devstack. -To provision Devstack Coursegraph with data from Devstack LMS, run:: +Tutor +***** + +The `CourseGraph plugin for Tutor`_ makes it easy to install, configure, and run CourseGraph for local development. + +Devstack +******** + +CourseGraph is included as an "extra" component in the `Open edX Devstack`_. That is, it is not run or provisioned by default, but can be enabled on-demand. + +To provision Devstack CourseGraph with data from Devstack LMS, run:: make dev.provision.coursegraph -Coursegraph should now be accessible at http://localhost:7474 with the username ``neo4j`` and the password ``edx``. +CourseGraph should now be accessible at http://localhost:7474 with the username ``neo4j`` and the password ``edx``. -Under the hood, the provisioning command just invokes ``dump_to_neo4j`` on your LMS, pointed at your Coursegraph. The provisioning command can be run again at any point in the future to refresh Coursegraph with new LMS data. The data in Coursegraph will persist unless you explicitly destroy it (as noted below). +Under the hood, the provisioning command just invokes ``dump_to_neo4j`` on your LMS, pointed at your CourseGraph. The provisioning command can be run again at any point in the future to refresh CourseGraph with new LMS data. The data in CourseGraph will persist unless you explicitly destroy it (as noted below). -Other Devstack Coursegraph commands include:: +Other Devstack CourseGraph commands include:: make dev.up.coursegraph # Bring up the container (without re-provisioning). make dev.down.coursegraph # Stop and remove the container. @@ -48,7 +65,7 @@ The above commands should be run in your ``devstack`` folder, and they assume th Querying Coursegraph ==================== -Coursegraph is queryable using the `Cypher`_ query language. Open edX learning content is represented in Neo4j using a straightforward scheme: +CourseGraph is queryable using the `Cypher`_ query language. Open edX learning content is represented in Neo4j using a straightforward scheme: * A node is an XBlock usage. @@ -97,3 +114,7 @@ In a given course, which units contain problems with custom Python grading code? c.course_key = '' RETURN u.location + +You can see many more examples of useful CourseGraph queries on the `query archive wiki page`_. + +.. _query archive wiki page: https://openedx.atlassian.net/wiki/spaces/COMM/pages/3273228388/Useful+CourseGraph+Queries From 21d57ed0ab9d62e8f41a498e39a51b844d18d052 Mon Sep 17 00:00:00 2001 From: Binod Pant Date: Tue, 29 Mar 2022 13:08:56 -0400 Subject: [PATCH 44/52] feat: post handler to sync provider_data (#30107) * feat: post handler to sync provider_data this allows us to read provider_data metadata from a remote metadata url. reuses code from the task that currently processes all proiderconfigs in a batch ENT-5482 * feat: lint fixes * test: add test for sync_provider_data * test: add case for update * fix: lint fix * fix: lint fix * feat: use exc_info to report error better * feat: update log message --- .../tests/test_samlproviderdata.py | 44 +++++++++++-- .../samlproviderdata/views.py | 58 ++++++++++++++-- common/djangoapps/third_party_auth/tasks.py | 36 ++-------- common/djangoapps/third_party_auth/utils.py | 66 ++++++++++++++++++- 4 files changed, 163 insertions(+), 41 deletions(-) diff --git a/common/djangoapps/third_party_auth/samlproviderdata/tests/test_samlproviderdata.py b/common/djangoapps/third_party_auth/samlproviderdata/tests/test_samlproviderdata.py index 2ceb1e968e..7607ee5dd9 100644 --- a/common/djangoapps/third_party_auth/samlproviderdata/tests/test_samlproviderdata.py +++ b/common/djangoapps/third_party_auth/samlproviderdata/tests/test_samlproviderdata.py @@ -1,18 +1,20 @@ # pylint: disable=missing-module-docstring import copy -import pytz -from uuid import uuid4 # lint-amnesty, pylint: disable=wrong-import-order from datetime import datetime # lint-amnesty, pylint: disable=wrong-import-order +from unittest import mock +from uuid import uuid4 # lint-amnesty, pylint: disable=wrong-import-order + +import pytz from django.contrib.sites.models import Site from django.urls import reverse from django.utils.http import urlencode +from enterprise.constants import ENTERPRISE_ADMIN_ROLE, ENTERPRISE_LEARNER_ROLE +from enterprise.models import EnterpriseCustomer, EnterpriseCustomerIdentityProvider from rest_framework import status from rest_framework.test import APITestCase -from enterprise.models import EnterpriseCustomer, EnterpriseCustomerIdentityProvider -from enterprise.constants import ENTERPRISE_ADMIN_ROLE, ENTERPRISE_LEARNER_ROLE from common.djangoapps.student.tests.factories import UserFactory -from common.djangoapps.third_party_auth.models import SAMLProviderData, SAMLProviderConfig +from common.djangoapps.third_party_auth.models import SAMLProviderConfig, SAMLProviderData from common.djangoapps.third_party_auth.tests.samlutils import set_jwt_cookie from common.djangoapps.third_party_auth.tests.utils import skip_unless_thirdpartyauth from common.djangoapps.third_party_auth.utils import convert_saml_slug_provider_id @@ -180,3 +182,35 @@ class SAMLProviderDataTests(APITestCase): set_jwt_cookie(self.client, self.user, [(ENTERPRISE_ADMIN_ROLE, BAD_ENTERPRISE_ID)]) response = self.client.get(url, format='json') assert response.status_code == status.HTTP_403_FORBIDDEN + + @mock.patch('common.djangoapps.third_party_auth.samlproviderdata.views.fetch_metadata_xml') + @mock.patch('common.djangoapps.third_party_auth.samlproviderdata.views.parse_metadata_xml') + def test_sync_one_provider_data_success(self, mock_parse, mock_fetch): + """ + POST auth/saml/v0/provider_data/sync_provider_data -d data + """ + mock_fetch.return_value = 'tag' + public_key = 'askdjf;sakdjfs;adkfjas;dkfjas;dkfjas;dlkfj' + sso_url = 'https://fake-test.id' + expires_at = datetime.now() + mock_parse.return_value = (public_key, sso_url, expires_at) + url = reverse('saml_provider_data-sync-provider-data') + data = { + 'entity_id': 'http://entity-id-1', + 'metadata_url': 'http://a-url', + 'enterprise_customer_uuid': ENTERPRISE_ID, + } + SAMLProviderData.objects.all().delete() + orig_count = SAMLProviderData.objects.count() + + response = self.client.post(url, data) + + assert response.status_code == status.HTTP_201_CREATED + assert response.data == " Created new record for SAMLProviderData for entityID http://entity-id-1" + assert SAMLProviderData.objects.count() == orig_count + 1 + + # should only update this time + response = self.client.post(url, data) + assert response.status_code == status.HTTP_200_OK + assert response.data == (" Updated existing SAMLProviderData for entityID http://entity-id-1") + assert SAMLProviderData.objects.count() == orig_count + 1 diff --git a/common/djangoapps/third_party_auth/samlproviderdata/views.py b/common/djangoapps/third_party_auth/samlproviderdata/views.py index 43c24db812..c3551cd656 100644 --- a/common/djangoapps/third_party_auth/samlproviderdata/views.py +++ b/common/djangoapps/third_party_auth/samlproviderdata/views.py @@ -1,21 +1,32 @@ """ Viewset for auth/saml/v0/samlproviderdata """ +import logging -from django.shortcuts import get_object_or_404 from django.http import Http404 +from django.shortcuts import get_object_or_404 from edx_rbac.mixins import PermissionRequiredMixin from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication -from rest_framework import permissions, viewsets -from rest_framework.authentication import SessionAuthentication -from rest_framework.exceptions import ParseError - from enterprise.models import EnterpriseCustomerIdentityProvider -from common.djangoapps.third_party_auth.utils import validate_uuid4_string, convert_saml_slug_provider_id +from rest_framework import permissions, status, viewsets +from rest_framework.authentication import SessionAuthentication +from rest_framework.decorators import action +from rest_framework.exceptions import ParseError +from rest_framework.response import Response + +from common.djangoapps.third_party_auth.utils import ( + convert_saml_slug_provider_id, + create_or_update_saml_provider_data, + fetch_metadata_xml, + parse_metadata_xml, + validate_uuid4_string +) from ..models import SAMLProviderConfig, SAMLProviderData from .serializers import SAMLProviderDataSerializer +log = logging.getLogger(__name__) + class SAMLProviderDataMixin: authentication_classes = [JwtAuthentication, SessionAuthentication] @@ -36,6 +47,7 @@ class SAMLProviderDataViewSet(PermissionRequiredMixin, SAMLProviderDataMixin, vi POST /auth/saml/v0/provider_data/ -d postData (must contain 'enterprise_customer_uuid') DELETE /auth/saml/v0/provider_data/:pk -d postData (must contain 'enterprise_customer_uuid') PATCH /auth/saml/v0/provider_data/:pk -d postData (must contain 'enterprise_customer_uuid') + POST /auth/saml/v0/provider_data/sync_provider_data (fetches metadata info from metadata url provided) """ permission_required = 'enterprise.can_access_admin_dashboard' @@ -81,3 +93,37 @@ class SAMLProviderDataViewSet(PermissionRequiredMixin, SAMLProviderDataMixin, vi Retrieve an EnterpriseCustomer to do auth against """ return self.requested_enterprise_uuid + + @action(detail=False, methods=['post']) + def sync_provider_data(self, request): + """ + Creates or updates a SAMProviderData record using info fetched from remote SAML metadata + For now we will require entityID but in future we will enhance this to try and extract entityID + from the metadata file, and make entityId optional, and return error response if there are + multiple entityIDs listed so that the user can choose and retry with a specified entityID + """ + entity_id = request.POST.get('entity_id') + metadata_url = request.POST.get('metadata_url') + if not entity_id: + return Response('entity_id is required!', status.HTTP_400_BAD_REQUEST) + if not metadata_url: + return Response('metadata_url is required!', status.HTTP_400_BAD_REQUEST) + + # part 1: fetch information from remote metadata based on metadataUrl in samlproviderconfig + xml = fetch_metadata_xml(metadata_url) + + # part 2: create/update samlproviderdata + log.info("Processing IdP with entityID %s", entity_id) + public_key, sso_url, expires_at = parse_metadata_xml(xml, entity_id) + changed = create_or_update_saml_provider_data(entity_id, public_key, sso_url, expires_at) + if changed: + str_message = f" Created new record for SAMLProviderData for entityID {entity_id}" + log.info(str_message) + response = str_message + http_status = status.HTTP_201_CREATED + else: + str_message = f" Updated existing SAMLProviderData for entityID {entity_id}" + log.info(str_message) + response = str_message + http_status = status.HTTP_200_OK + return Response(response, status=http_status) diff --git a/common/djangoapps/third_party_auth/tasks.py b/common/djangoapps/third_party_auth/tasks.py index 2b29ce20cc..88b118a689 100644 --- a/common/djangoapps/third_party_auth/tasks.py +++ b/common/djangoapps/third_party_auth/tasks.py @@ -7,13 +7,16 @@ import logging import requests from celery import shared_task -from django.utils.timezone import now from edx_django_utils.monitoring import set_code_owner_attribute from lxml import etree from requests import exceptions -from common.djangoapps.third_party_auth.models import SAMLConfiguration, SAMLProviderConfig, SAMLProviderData -from common.djangoapps.third_party_auth.utils import MetadataParseError, parse_metadata_xml +from common.djangoapps.third_party_auth.models import SAMLConfiguration, SAMLProviderConfig +from common.djangoapps.third_party_auth.utils import ( + MetadataParseError, + create_or_update_saml_provider_data, + parse_metadata_xml, +) log = logging.getLogger(__name__) @@ -85,7 +88,7 @@ def fetch_saml_metadata(): for entity_id in entity_ids: log.info("Processing IdP with entityID %s", entity_id) public_key, sso_url, expires_at = parse_metadata_xml(xml, entity_id) - changed = _update_data(entity_id, public_key, sso_url, expires_at) + changed = create_or_update_saml_provider_data(entity_id, public_key, sso_url, expires_at) if changed: log.info(f"→ Created new record for SAMLProviderData for entityID {entity_id}") num_updated += 1 @@ -124,28 +127,3 @@ def fetch_saml_metadata(): # Return counts for total, skipped, attempted, updated, and failed, along with any failure messages return num_total, num_skipped, num_attempted, num_updated, len(failure_messages), failure_messages - - -def _update_data(entity_id, public_key, sso_url, expires_at): - """ - Update/Create the SAMLProviderData for the given entity ID. - Return value: - False if nothing has changed and existing data's "fetched at" timestamp is just updated. - True if a new record was created. (Either this is a new provider or something changed.) - """ - data_obj = SAMLProviderData.current(entity_id) - fetched_at = now() - if data_obj and (data_obj.public_key == public_key and data_obj.sso_url == sso_url): - data_obj.expires_at = expires_at - data_obj.fetched_at = fetched_at - data_obj.save() - return False - else: - SAMLProviderData.objects.create( - entity_id=entity_id, - fetched_at=fetched_at, - expires_at=expires_at, - sso_url=sso_url, - public_key=public_key, - ) - return True diff --git a/common/djangoapps/third_party_auth/utils.py b/common/djangoapps/third_party_auth/utils.py index 3d411bb63a..8517af0328 100644 --- a/common/djangoapps/third_party_auth/utils.py +++ b/common/djangoapps/third_party_auth/utils.py @@ -3,29 +3,68 @@ Utility functions for third_party_auth """ import datetime +import logging from uuid import UUID import dateutil.parser import pytz +import requests from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user +from django.utils.timezone import now from enterprise.models import EnterpriseCustomerIdentityProvider, EnterpriseCustomerUser from lxml import etree from onelogin.saml2.utils import OneLogin_Saml2_Utils +from requests import exceptions from social_core.pipeline.social_auth import associate_by_email -from common.djangoapps.third_party_auth.models import OAuth2ProviderConfig +from common.djangoapps.third_party_auth.models import OAuth2ProviderConfig, SAMLProviderData from openedx.core.djangolib.markup import Text from . import provider SAML_XML_NS = 'urn:oasis:names:tc:SAML:2.0:metadata' # The SAML Metadata XML namespace +log = logging.getLogger(__name__) + class MetadataParseError(Exception): """ An error occurred while parsing the SAML metadata from an IdP """ pass # lint-amnesty, pylint: disable=unnecessary-pass +def fetch_metadata_xml(url): + """ + Fetches IDP metadata from provider url + Returns: xml document + """ + try: + log.info("Fetching %s", url) + if not url.lower().startswith('https'): + log.warning("This SAML metadata URL is not secure! It should use HTTPS. (%s)", url) + response = requests.get(url, verify=True) # May raise HTTPError or SSLError or ConnectionError + response.raise_for_status() # May raise an HTTPError + + try: + parser = etree.XMLParser(remove_comments=True) + xml = etree.fromstring(response.content, parser) + except etree.XMLSyntaxError: # lint-amnesty, pylint: disable=try-except-raise + raise + # TODO: Can use OneLogin_Saml2_Utils to validate signed XML if anyone is using that + return xml + except (exceptions.SSLError, exceptions.HTTPError, exceptions.RequestException, MetadataParseError) as error: + # Catch and process exception in case of errors during fetching and processing saml metadata. + # Here is a description of each exception. + # SSLError is raised in case of errors caused by SSL (e.g. SSL cer verification failure etc.) + # HTTPError is raised in case of unexpected status code (e.g. 500 error etc.) + # RequestException is the base exception for any request related error that "requests" lib raises. + # MetadataParseError is raised if there is error in the fetched meta data (e.g. missing @entityID etc.) + log.exception(str(error), exc_info=error) + raise error + except etree.XMLSyntaxError as error: + log.exception(str(error), exc_info=error) + raise error + + def parse_metadata_xml(xml, entity_id): """ Given an XML document containing SAML 2.0 metadata, parse it and return a tuple of @@ -125,6 +164,31 @@ def get_user_from_email(details): return None +def create_or_update_saml_provider_data(entity_id, public_key, sso_url, expires_at): + """ + Update/Create the SAMLProviderData for the given entity ID. + Return value: + False if nothing has changed and existing data's "fetched at" timestamp is just updated. + True if a new record was created. (Either this is a new provider or something changed.) + """ + data_obj = SAMLProviderData.current(entity_id) + fetched_at = now() + if data_obj and (data_obj.public_key == public_key and data_obj.sso_url == sso_url): + data_obj.expires_at = expires_at + data_obj.fetched_at = fetched_at + data_obj.save() + return False + else: + SAMLProviderData.objects.create( + entity_id=entity_id, + fetched_at=fetched_at, + expires_at=expires_at, + sso_url=sso_url, + public_key=public_key, + ) + return True + + def convert_saml_slug_provider_id(provider): # lint-amnesty, pylint: disable=redefined-outer-name """ Provider id is stored with the backend type prefixed to it (ie "saml-") From 4a7719e73f2f92d281344fd765cc2b5841cfab73 Mon Sep 17 00:00:00 2001 From: John Nagro Date: Tue, 29 Mar 2022 13:38:43 -0400 Subject: [PATCH 45/52] fix: release edx-enterprise 3.41.7 (#30145) - https://github.com/openedx/edx-enterprise/pull/1508 ENT-5595 ENT-5603 ENT-5605 --- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 961de0338c..a87ca56b48 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -31,7 +31,7 @@ django-storages<1.9 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==3.41.6 +edx-enterprise==3.41.7 # Newer versions need a more recent version of python-dateutil freezegun==0.3.12 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index e7e9a8f463..90f4d2291e 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -452,7 +452,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.6 +edx-enterprise==3.41.7 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index c100cc2f89..9ea58a61ff 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -557,7 +557,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.6 +edx-enterprise==3.41.7 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index f2b91c69b2..cc57fd7594 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -541,7 +541,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.6 +edx-enterprise==3.41.7 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt From fd60d8c8cf96f145efedb0b5d4602e03f5ea3e35 Mon Sep 17 00:00:00 2001 From: Saad Yousaf Date: Wed, 30 Mar 2022 11:31:45 +0500 Subject: [PATCH 46/52] fix: revert discussion xblock from MFE view to legacy view. (#30141) Co-authored-by: SaadYousaf --- .../tests/test_discussion_xblock.py | 46 +++---------------- .../xblock_discussion/__init__.py | 16 ------- 2 files changed, 7 insertions(+), 55 deletions(-) diff --git a/lms/djangoapps/courseware/tests/test_discussion_xblock.py b/lms/djangoapps/courseware/tests/test_discussion_xblock.py index 4e191f052c..aa77725c2c 100644 --- a/lms/djangoapps/courseware/tests/test_discussion_xblock.py +++ b/lms/djangoapps/courseware/tests/test_discussion_xblock.py @@ -12,9 +12,7 @@ import uuid from unittest import mock import ddt -from django.test import override_settings from django.urls import reverse -from edx_toggles.toggles.testutils import override_waffle_flag from opaque_keys.edx.keys import CourseKey from web_fragments.fragment import Fragment from xblock.field_data import DictFieldData @@ -26,7 +24,6 @@ from xmodule.modulestore.tests.factories import ItemFactory, ToyCourseFactory from lms.djangoapps.course_api.blocks.tests.helpers import deserialize_usage_key from lms.djangoapps.courseware.module_render import get_module_for_descriptor_internal from lms.djangoapps.courseware.tests.helpers import XModuleRenderingTestBase -from lms.djangoapps.discussion.toggles import ENABLE_DISCUSSIONS_MFE from common.djangoapps.student.tests.factories import CourseEnrollmentFactory, UserFactory @@ -308,34 +305,6 @@ class TestXBlockInCourse(SharedModuleStoreTestCase): assert 'data-user-create-comment="false"' in html assert 'data-user-create-subcomment="false"' in html - @override_settings(DISCUSSIONS_MICROFRONTEND_URL="http://test.url") - @override_waffle_flag(ENABLE_DISCUSSIONS_MFE, True) - def test_embed_mfe_in_course(self): - """ - Test that the xblock embeds the MFE UI when the flag is enabled - """ - discussion_xblock = get_module_for_descriptor_internal( - user=self.user, - descriptor=self.discussion, - student_data=mock.Mock(name='student_data'), - course_id=self.course.id, - track_function=mock.Mock(name='track_function'), - request_token='request_token', - ) - - fragment = discussion_xblock.render('student_view') - html = fragment.content - self.assertInHTML( - """ - " - ).format(src=mfe_url, title=_("Discussions"))) - fragment.add_css( - """ - #discussions-mfe-tab-embed { - width: 100%; - height: 800px; - border: none; - } - """ - ) - return fragment - self.add_resource_urls(fragment) login_msg = '' From f52c24d445fff9a3430d2e88922119549727e722 Mon Sep 17 00:00:00 2001 From: Sameen Fatima <55431213+sameenfatima78@users.noreply.github.com> Date: Wed, 30 Mar 2022 14:09:54 +0500 Subject: [PATCH 47/52] feat: Add authoring organization logo in API response for all learner enrolled programs (#30139) --- .../learner_dashboard/api/v0/tests/test_views.py | 3 ++- lms/djangoapps/learner_dashboard/api/v0/views.py | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lms/djangoapps/learner_dashboard/api/v0/tests/test_views.py b/lms/djangoapps/learner_dashboard/api/v0/tests/test_views.py index ad94339d67..7072235ef1 100644 --- a/lms/djangoapps/learner_dashboard/api/v0/tests/test_views.py +++ b/lms/djangoapps/learner_dashboard/api/v0/tests/test_views.py @@ -166,7 +166,8 @@ class TestProgramsView(SharedModuleStoreTestCase, ProgramCacheMixin): title='Journey to cooking', type='MicroMasters', authoring_organizations=[{ - 'key': 'MAX' + 'key': 'MAX', + 'logo_image_url': 'http://test.org/media/organization/logos/test-logo.png' }], ) cls.site = SiteFactory(domain='test.localhost') diff --git a/lms/djangoapps/learner_dashboard/api/v0/views.py b/lms/djangoapps/learner_dashboard/api/v0/views.py index 256ba7295b..7a26adc201 100644 --- a/lms/djangoapps/learner_dashboard/api/v0/views.py +++ b/lms/djangoapps/learner_dashboard/api/v0/views.py @@ -134,7 +134,12 @@ class Programs(APIView): """ transformed_authoring_organizations = [] for authoring_organization in authoring_organizations: - transformed_authoring_organizations.append({'key': authoring_organization['key']}) + transformed_authoring_organizations.append( + { + 'key': authoring_organization['key'], + 'logo_image_url': authoring_organization['logo_image_url'] + } + ) return transformed_authoring_organizations From 18e3516b36aef94da9d259bf1574103764ceb300 Mon Sep 17 00:00:00 2001 From: Ahtisham Shahid Date: Wed, 30 Mar 2022 15:34:09 +0500 Subject: [PATCH 48/52] feat: Added permissions for course staff to discussion MFE (#30143) --- lms/djangoapps/discussion/tests/test_views.py | 2 +- lms/djangoapps/discussion/views.py | 20 +++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/lms/djangoapps/discussion/tests/test_views.py b/lms/djangoapps/discussion/tests/test_views.py index 0cec540d3c..a44bd19775 100644 --- a/lms/djangoapps/discussion/tests/test_views.py +++ b/lms/djangoapps/discussion/tests/test_views.py @@ -2287,7 +2287,7 @@ class ForumMFETestCase(ForumsEnableMixin, SharedModuleStoreTestCase): self.staff_user = AdminFactory.create() CourseEnrollmentFactory.create(user=self.user, course_id=self.course.id) - @ddt.data(*itertools.product(("http://test.url", None), (True, False), (True, False))) + @ddt.data(*itertools.product(("http://test.url", None), (True, False), (True, True))) @ddt.unpack def test_staff_user(self, mfe_url, toggle_enabled, is_staff): """ diff --git a/lms/djangoapps/discussion/views.py b/lms/djangoapps/discussion/views.py index 84cf8ce119..91a41fd06e 100644 --- a/lms/djangoapps/discussion/views.py +++ b/lms/djangoapps/discussion/views.py @@ -61,6 +61,7 @@ from openedx.core.djangoapps.django_comment_common.models import CourseDiscussio from openedx.core.djangoapps.django_comment_common.utils import ThreadContext from openedx.core.djangoapps.plugin_api.views import EdxFragmentView from openedx.features.course_duration_limits.access import generate_course_expired_fragment +from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole, GlobalStaff User = get_user_model() log = logging.getLogger("edx.discussions") @@ -703,7 +704,10 @@ def followed_threads(request, course_key, user_id): raise Http404 # lint-amnesty, pylint: disable=raise-missing-from -def _discussions_mfe_context(query_params: Dict, course_key: CourseKey, legacy_only_view=False) -> Optional[Dict]: +def _discussions_mfe_context(query_params: Dict, + course_key: CourseKey, + is_educator_or_staff=False, + legacy_only_view=False) -> Optional[Dict]: """ Returns the context for rendering the MFE banner and MFE. @@ -723,7 +727,7 @@ def _discussions_mfe_context(query_params: Dict, course_key: CourseKey, legacy_o # and if the current view isn't only that's only supported by the legacy view show_mfe = ( query_params.get("discussions_experience", "").lower() != "legacy" - and discussions_mfe_enabled + and (discussions_mfe_enabled and is_educator_or_staff) and not legacy_only_view ) forum_url = reverse("forum_form_discussion", args=[course_key]) @@ -733,11 +737,18 @@ def _discussions_mfe_context(query_params: Dict, course_key: CourseKey, legacy_o "mfe_url": f"{forum_url}?discussions_experience=new", "share_feedback_url": settings.DISCUSSIONS_MFE_FEEDBACK_URL, "course_key": course_key, - "show_banner": discussions_mfe_enabled, + "show_banner": (discussions_mfe_enabled and is_educator_or_staff), "discussions_mfe_url": mfe_url, } +def is_course_staff(course_key: CourseKey, user: User): + """ + Check if user has course instructor or course staff role. + """ + return CourseInstructorRole(course_key).has_user(user) or CourseStaffRole(course_key).has_user(user) + + class DiscussionBoardFragmentView(EdxFragmentView): """ Component implementation of the discussion board. @@ -767,7 +778,8 @@ class DiscussionBoardFragmentView(EdxFragmentView): course_key = CourseKey.from_string(course_id) # Force using the legacy view if a user profile is requested or the URL contains a specific topic or thread force_legacy_view = (profile_page_context or thread_id or discussion_id) - mfe_context = _discussions_mfe_context(request.GET, course_key, force_legacy_view) + is_educator_or_staff = is_course_staff(course_key, request.user) or GlobalStaff().has_user(request.user) + mfe_context = _discussions_mfe_context(request.GET, course_key, is_educator_or_staff, force_legacy_view) if mfe_context["show_mfe"]: fragment = Fragment(render_to_string('discussion/discussion_mfe_embed.html', mfe_context)) fragment.add_css( From c88db037fa7472f857c215fea562b40f1c6b45d7 Mon Sep 17 00:00:00 2001 From: Mubbshar Anwar <78487564+mubbsharanwar@users.noreply.github.com> Date: Wed, 30 Mar 2022 18:28:21 +0500 Subject: [PATCH 49/52] chore: save for later admin (#30152) register save_for_later app models in django admin. VAN-887 --- lms/djangoapps/save_for_later/admin.py | 29 ++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 lms/djangoapps/save_for_later/admin.py diff --git a/lms/djangoapps/save_for_later/admin.py b/lms/djangoapps/save_for_later/admin.py new file mode 100644 index 0000000000..aa68212463 --- /dev/null +++ b/lms/djangoapps/save_for_later/admin.py @@ -0,0 +1,29 @@ +""" Django admin pages for save_for_later app """ + +from django.contrib import admin + +from lms.djangoapps.save_for_later.models import SavedCourse, SavedProgram + + +class SavedCourseAdmin(admin.ModelAdmin): + """ + Admin for the Saved Course table. + """ + + list_display = ['email', 'course_id'] + + search_fields = ['email', 'course_id'] + + +class SavedProgramAdmin(admin.ModelAdmin): + """ + Admin for the Saved Program table. + """ + + list_display = ['email', 'program_uuid'] + + search_fields = ['email', 'program_uuid'] + + +admin.site.register(SavedCourse, SavedCourseAdmin) +admin.site.register(SavedProgram, SavedProgramAdmin) From f2a44ada4155032e3e7e8afc1ea0d723bcf79716 Mon Sep 17 00:00:00 2001 From: John Nagro Date: Wed, 30 Mar 2022 11:56:42 -0400 Subject: [PATCH 50/52] fix: release edx-enterprise 3.41.9 (#30153) - https://github.com/openedx/edx-enterprise/pull/1513 - https://github.com/openedx/edx-enterprise/pull/1515 --- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index a87ca56b48..7dfe2afcb4 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -31,7 +31,7 @@ django-storages<1.9 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==3.41.7 +edx-enterprise==3.41.9 # Newer versions need a more recent version of python-dateutil freezegun==0.3.12 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 90f4d2291e..adf31d7fb8 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -452,7 +452,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.7 +edx-enterprise==3.41.9 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 9ea58a61ff..8c39c61c9a 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -557,7 +557,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.7 +edx-enterprise==3.41.9 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index cc57fd7594..459e4fba56 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -541,7 +541,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.7 +edx-enterprise==3.41.9 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt From e1d9746d61946bbc34cdeb3a3e1a38e38e680894 Mon Sep 17 00:00:00 2001 From: Ghassan Maslamani Date: Wed, 30 Mar 2022 21:45:08 +0300 Subject: [PATCH 51/52] fix: make bulk_email send email task handles a retryable exception (#29080) * fix: make bulk_email send email task handles a retryable exception According to bulk_email and SMTP doc, when an error code is between (400-499) it can be retried after sometime, this commit adds a new type of exception called SMTPSenderRefused, so it can be retried **if** it's code fails under above constraint. * test: add test to handle SMTPSenderResfused Co-authored-by: Justin Hynes --- lms/djangoapps/bulk_email/tasks.py | 9 +++++---- lms/djangoapps/bulk_email/tests/test_tasks.py | 7 ++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lms/djangoapps/bulk_email/tasks.py b/lms/djangoapps/bulk_email/tasks.py index d6b8393f47..4d86edcc10 100644 --- a/lms/djangoapps/bulk_email/tasks.py +++ b/lms/djangoapps/bulk_email/tasks.py @@ -10,7 +10,7 @@ import re import time from collections import Counter from datetime import datetime -from smtplib import SMTPConnectError, SMTPDataError, SMTPException, SMTPServerDisconnected +from smtplib import SMTPConnectError, SMTPDataError, SMTPException, SMTPServerDisconnected, SMTPSenderRefused from time import sleep from boto.exception import AWSConnectionError @@ -87,12 +87,13 @@ LIMITED_RETRY_ERRORS = ( # An example is if email is being sent too quickly, but may succeed if sent # more slowly. When caught by a task, it triggers an exponential backoff and retry. # Retries happen continuously until the email is sent. -# Note that the SMTPDataErrors here are only those within the 4xx range. +# Note that the (SMTPDataErrors and SMTPSenderRefused) here are only those within the 4xx range. # Those not in this range (i.e. in the 5xx range) are treated as hard failures # and thus like SINGLE_EMAIL_FAILURE_ERRORS. INFINITE_RETRY_ERRORS = ( SESMaxSendingRateExceededError, # Your account's requests/second limit has been exceeded. SMTPDataError, + SMTPSenderRefused, ) # Errors that are known to indicate an inability to send any more emails, @@ -565,11 +566,11 @@ def _send_course_email(entry_id, email_id, to_list, global_email_context, subtas f"{recipient_num}/{total_recipients}, Recipient UserId: {current_recipient['pk']}" ) message.send() - except SMTPDataError as exc: + except (SMTPDataError, SMTPSenderRefused) as exc: # According to SMTP spec, we'll retry error codes in the 4xx range. 5xx range indicates hard failure. total_recipients_failed += 1 log.exception( - f"BulkEmail ==> Status: Failed(SMTPDataError), Task: {parent_task_id}, SubTask: {task_id}, " + f"BulkEmail ==> Status: Failed({exc.smtp_error}), Task: {parent_task_id}, SubTask: {task_id}, " f"EmailId: {email_id}, Recipient num: {recipient_num}/{total_recipients}, Recipient UserId: " f"{current_recipient['pk']}" ) diff --git a/lms/djangoapps/bulk_email/tests/test_tasks.py b/lms/djangoapps/bulk_email/tests/test_tasks.py index 51f7b4161c..e73db046aa 100644 --- a/lms/djangoapps/bulk_email/tests/test_tasks.py +++ b/lms/djangoapps/bulk_email/tests/test_tasks.py @@ -11,7 +11,7 @@ from datetime import datetime from dateutil.relativedelta import relativedelta import json # lint-amnesty, pylint: disable=wrong-import-order from itertools import chain, cycle, repeat # lint-amnesty, pylint: disable=wrong-import-order -from smtplib import SMTPAuthenticationError, SMTPConnectError, SMTPDataError, SMTPServerDisconnected # lint-amnesty, pylint: disable=wrong-import-order +from smtplib import SMTPAuthenticationError, SMTPConnectError, SMTPDataError, SMTPServerDisconnected, SMTPSenderRefused # lint-amnesty, pylint: disable=wrong-import-order from unittest.mock import Mock, patch # lint-amnesty, pylint: disable=wrong-import-order from uuid import uuid4 # lint-amnesty, pylint: disable=wrong-import-order import pytest @@ -411,6 +411,11 @@ class TestBulkEmailInstructorTask(InstructorTaskCourseTestCase): def test_retry_after_smtp_throttling_error(self): self._test_retry_after_unlimited_retry_error(SMTPDataError(455, "Throttling: Sending rate exceeded")) + def test_retry_after_smtp_sender_refused_error(self): + self._test_retry_after_unlimited_retry_error( + SMTPSenderRefused(421, "Throttling: Sending rate exceeded", self.instructor.email) + ) + def test_retry_after_ses_throttling_error(self): self._test_retry_after_unlimited_retry_error( SESMaxSendingRateExceededError(455, "Throttling: Sending rate exceeded") From 6352fed661bc15021c01dee018152b8f2aacf127 Mon Sep 17 00:00:00 2001 From: John Nagro Date: Wed, 30 Mar 2022 17:07:25 -0400 Subject: [PATCH 52/52] feat: release edx-enterprise 3.41.10 (#30155) - feat: management command to backfill missing fks - https://github.com/openedx/edx-enterprise/pull/1516 ENT-5605 --- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 7dfe2afcb4..fa3412bb01 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -31,7 +31,7 @@ django-storages<1.9 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==3.41.9 +edx-enterprise==3.41.10 # Newer versions need a more recent version of python-dateutil freezegun==0.3.12 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index adf31d7fb8..27b8b7d34f 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -452,7 +452,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.9 +edx-enterprise==3.41.10 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 8c39c61c9a..37f2f3b1a2 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -557,7 +557,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.9 +edx-enterprise==3.41.10 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 459e4fba56..c3a5c59f85 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -541,7 +541,7 @@ edx-drf-extensions==8.0.1 # edx-rbac # edx-when # edxval -edx-enterprise==3.41.9 +edx-enterprise==3.41.10 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt