diff --git a/openedx/core/lib/request_utils.py b/openedx/core/lib/request_utils.py index dfeb2d4356..7255052af2 100644 --- a/openedx/core/lib/request_utils.py +++ b/openedx/core/lib/request_utils.py @@ -102,18 +102,104 @@ class CookieMonitoringMiddleware(MiddlewareMixin): Don't log contents of cookies because that might cause a security issue. We just want to see if any cookies are growing out of control. + + A useful NRQL Query: + SELECT count(*), max(`cookies.max.group.size`) from Transaction FACET + `cookies.max.group.name` + + SELECT * FROM Transaction WHERE cookies_total_size > 6000 + + Attributes that are added by this middleware: + + cookies..name: The name of the Nth largest cookie + cookies..size: The size of the Nth largest cookie + cookies..group..name: The name of the Nth largest cookie. + cookies.group..size: The size of the Nth largest cookie group. + cookies.max.name: The name of the largest cookie sent by the user. + cookies.max.size: The size of the largest cookie sent by the user. + cookies.max.group.name: The name of the largest group of cookies. A single cookie + counts as a group of one for this calculation. + cookies.max.group.size: The sum total size of all the cookies in the largest group. + cookies_total_size: The sum total size of all cookies in this request. + + Related Settings: + + - `request_utils.capture_cookie_sizes` is the waffle flag that control whether this + middleware logs anything or not. + + - TOP_N_COOKIES_CAPTURED(Default: 5) controls how many cookies to log. + - TOP_N_COOKIE_GROUPS_CAPTURED(Default: 5): controls how many cookie groups to capture. + + """ if not CAPTURE_COOKIE_SIZES.is_enabled(): return - cookie_names_to_size = { - name: len(value) - for name, value in request.COOKIES.items() - } - for name, size in cookie_names_to_size.items(): - attribute_name = 'cookies.{}.size'.format(name) - set_custom_attribute(attribute_name, size) - log.debug(u'%s = %d', attribute_name, size) + # Capture the N largest cookies + top_n_cookies_captured = getattr(settings, "TOP_N_COOKIES_CAPTURED", 5) + top_n_cookie_groups_captured = getattr(settings, "TOP_N_COOKIE_GROUPS_CAPTURED", 5) + + cookie_names_to_size = {} + cookie_groups_to_size = {} + + for name, value in request.COOKIES.items(): + # Get cookie size for all cookies. + cookie_size = len(value) + cookie_names_to_size[name] = cookie_size + + # Group cookies by their prefix seperated by a period or underscore + grouping_name = re.split('[._]', name, 1)[0] + if grouping_name and grouping_name != name: + # Add or update the size for this group. + cookie_groups_to_size[grouping_name] = cookie_groups_to_size.get(grouping_name, 0) + cookie_size + + max_cookie_name = max(cookie_names_to_size, key=lambda name: cookie_names_to_size[name]) + max_cookie_size = cookie_names_to_size[max_cookie_name] + + max_group_cookie_name = max(cookie_groups_to_size, key=lambda name: cookie_groups_to_size[name]) + max_group_cookie_size = cookie_groups_to_size[max_group_cookie_name] + + # If a single cookies is bigger than any group of cookies, we want max_group... to reflect that. + # Treating an individual cookie as a group of 1 for calculating the max. + if max_group_cookie_size < max_cookie_size: + max_group_cookie_name = max_cookie_name + max_group_cookie_size = max_cookie_size + + # Log only the top N biggest cookies. + top_n_cookies = sorted( + cookie_names_to_size, + key=lambda x: cookie_names_to_size[x], + reverse=True, + )[:top_n_cookies_captured] + for index, name in enumerate(top_n_cookies, start=1): + size = cookie_names_to_size[name] + name_attribute = 'cookies.{}.name'.format(index) + size_attribute = 'cookies.{}.size'.format(index) + + set_custom_attribute(name_attribute, name) + set_custom_attribute(size_attribute, size) + log.debug(u'%s = %d', name, size) + + # Log only the top N biggest groups. + top_n_cookie_groups = sorted( + cookie_groups_to_size, + key=lambda x: cookie_groups_to_size[x], + reverse=True, + )[:top_n_cookie_groups_captured] + + for index, name in enumerate(top_n_cookie_groups, start=1): + size = cookie_groups_to_size[name] + name_attribute = 'cookies.group.{}.name'.format(index) + size_attribute = 'cookies.group.{}.size'.format(index) + + set_custom_attribute(name_attribute, name) + set_custom_attribute(size_attribute, size) + log.debug(u'%s = %d', name, size) + + set_custom_attribute('cookies.max.name', max_cookie_name) + set_custom_attribute('cookies.max.size', max_cookie_size) + set_custom_attribute('cookies.max.group.name', max_group_cookie_name) + set_custom_attribute('cookies.max.group.size', max_group_cookie_size) total_cookie_size = sum(cookie_names_to_size.values()) set_custom_attribute('cookies_total_size', total_cookie_size) diff --git a/openedx/core/lib/tests/test_request_utils.py b/openedx/core/lib/tests/test_request_utils.py index b091dfd790..6c140b7f11 100644 --- a/openedx/core/lib/tests/test_request_utils.py +++ b/openedx/core/lib/tests/test_request_utils.py @@ -2,12 +2,18 @@ import unittest +from unittest.mock import Mock, patch, call from django.conf import settings from django.core.exceptions import SuspiciousOperation from django.test.client import RequestFactory -from openedx.core.lib.request_utils import get_request_or_stub, course_id_from_url, safe_get_host +from openedx.core.lib.request_utils import ( + get_request_or_stub, + course_id_from_url, + safe_get_host, + CookieMonitoringMiddleware, +) class RequestUtilTestCase(unittest.TestCase): @@ -83,3 +89,78 @@ class RequestUtilTestCase(unittest.TestCase): self.assertEqual(course_id.org, org) self.assertEqual(course_id.course, course) self.assertEqual(course_id.run, run) + + @patch("openedx.core.lib.request_utils.CAPTURE_COOKIE_SIZES") + @patch("openedx.core.lib.request_utils.set_custom_attribute") + def test_cookie_monitoring(self, mock_set_custom_attribute, mock_capture_cookie_sizes): + + mock_capture_cookie_sizes.is_enabled.return_value = True + middleware = CookieMonitoringMiddleware() + + mock_request = Mock() + mock_request.COOKIES = { + "a": "." * 100, + "_b": "." * 13, + "_c_": "." * 13, + "a.b": "." * 10, + "a.c": "." * 10, + "b.": "." * 13, + "b_a": "." * 15, + "b_c": "." * 15, + } + + middleware.process_request(mock_request) + + mock_set_custom_attribute.assert_has_calls([ + call('cookies.1.name', 'a'), + call('cookies.1.size', 100), + call('cookies.2.name', 'b_a'), + call('cookies.2.size', 15), + call('cookies.3.name', 'b_c'), + call('cookies.3.size', 15), + call('cookies.4.name', '_b'), + call('cookies.4.size', 13), + call('cookies.5.name', '_c_'), + call('cookies.5.size', 13), + call('cookies.group.1.name', 'b'), + call('cookies.group.1.size', 43), + call('cookies.group.2.name', 'a'), + call('cookies.group.2.size', 20), + call('cookies.max.name', 'a'), + call('cookies.max.size', 100), + call('cookies.max.group.name', 'a'), + call('cookies.max.group.size', 100), + call('cookies_total_size', 189), + ]) + + @patch("openedx.core.lib.request_utils.CAPTURE_COOKIE_SIZES") + @patch("openedx.core.lib.request_utils.set_custom_attribute") + def test_cookie_monitoring_max_group(self, mock_set_custom_attribute, mock_capture_cookie_sizes): + + mock_capture_cookie_sizes.is_enabled.return_value = True + middleware = CookieMonitoringMiddleware() + + mock_request = Mock() + mock_request.COOKIES = { + "a": "." * 10, + "b_a": "." * 15, + "b_c": "." * 20, + } + + middleware.process_request(mock_request) + + mock_set_custom_attribute.assert_has_calls([ + call('cookies.1.name', 'b_c'), + call('cookies.1.size', 20), + call('cookies.2.name', 'b_a'), + call('cookies.2.size', 15), + call('cookies.3.name', 'a'), + call('cookies.3.size', 10), + call('cookies.group.1.name', 'b'), + call('cookies.group.1.size', 35), + call('cookies.max.name', 'b_c'), + call('cookies.max.size', 20), + call('cookies.max.group.name', 'b'), + call('cookies.max.group.size', 35), + call('cookies_total_size', 45) + ])