Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add more cookie logging code.

Browse Source 
We want to be able to easily figure out what our biggest cookies are and we
want to also group cookies by prefix because certain services create multiple
cookies and then put unique identifiers in the cookie name.

For example braze cookie names use the following pattern:
    ab.storage.<userId>
    ab.storage.<deviceId>
    ab.storage.<sessionId>

In this case we want to group all the `ab` cookies together so we can see
their total size.

New attributes:

cookies.<group_prefix>.group.size: The size of a group of cookies. For example
	the sum of the size of all braze cookies would be the value of the
	`cookies.ab.group.size` attribute.
cookies.max.name: The name of the largest cookie sent by the user.
cookies.max.size: The size of the largest cookie sent by the user.
cookies.max.group.name: The name of the largest group of cookies. A single cookie
	counts as a group of one for this calculation.
cookies.max.group.size: The sum total size of all the cookies in the largest group.
This commit is contained in:
Feanil Patel
2021-01-13 16:37:05 -05:00
parent db152dda45
commit 1ae50fc466
2 changed files with 133 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
openedx/core/lib/request_utils.py
View File

@@ -102,19 +102,71 @@ class CookieMonitoringMiddleware(MiddlewareMixin):
Don't log contents of cookies because that might cause a security issue.
We just want to see if any cookies are growing out of control.
A useful NRQL Query:
SELECT count(*), max(`cookies.max.group.size`) from Transaction FACET
`cookies.max.group.name`
SELECT * FROM Transaction WHERE cookies_total_size > 6000
Attributes that are added by this middleware:
cookies.<cookie_name>.size: The size of a cookie by the given name.
cookies.<group_prefix>.group.size: The size of a group of cookies. For example
the sum of the size of all braze cookies would be the value of the
`cookies.ab.group.size` attribute.
cookies.max.name: The name of the largest cookie sent by the user.
cookies.max.size: The size of the largest cookie sent by the user.
cookies.max.group.name: The name of the largest group of cookies. A single cookie
counts as a group of one for this calculation.
cookies.max.group.size: The sum total size of all the cookies in the largest group.
cookies_total_size: The sum total size of all cookies in this request.
"""
if not CAPTURE_COOKIE_SIZES.is_enabled():
return
cookie_names_to_size = {
name: len(value)
for name, value in request.COOKIES.items()
}
cookie_names_to_size = {}
cookie_groups_to_size = {}
for name, value in request.COOKIES.items():
# Get cookie size for all cookies.
cookie_size = len(value)
cookie_names_to_size[name] = cookie_size
# Group cookies by their prefix seperated by a period or underscore
grouping_name = re.split('[._]', name, 1)[0]
if grouping_name and grouping_name != name:
# Add or update the size for this group.
cookie_groups_to_size[grouping_name] = cookie_groups_to_size.get(grouping_name, 0) + cookie_size
max_cookie_name = max(cookie_names_to_size, key=lambda name: cookie_names_to_size[name])
max_cookie_size = cookie_names_to_size[max_cookie_name]
max_group_cookie_name = max(cookie_groups_to_size, key=lambda name: cookie_groups_to_size[name])
max_group_cookie_size = cookie_groups_to_size[max_group_cookie_name]
# If a single cookies is bigger than any group of cookies, we want max_group... to reflect that.
# Treating an individual cookie as a group of 1 for calculating the max.
if max_group_cookie_size < max_cookie_size:
max_group_cookie_name = max_cookie_name
max_group_cookie_size = max_cookie_size
for name, size in cookie_names_to_size.items():
attribute_name = 'cookies.{}.size'.format(name)
set_custom_attribute(attribute_name, size)
log.debug(u'%s = %d', attribute_name, size)
for name, size in cookie_groups_to_size.items():
attribute_name = 'cookies.{}.group.size'.format(name)
set_custom_attribute(attribute_name, size)
log.debug(u'%s = %d', attribute_name, size)
set_custom_attribute('cookies.max.name', max_cookie_name)
set_custom_attribute('cookies.max.size', max_cookie_size)
set_custom_attribute('cookies.max.group.name', max_group_cookie_name)
set_custom_attribute('cookies.max.group.size', max_group_cookie_size)
total_cookie_size = sum(cookie_names_to_size.values())
set_custom_attribute('cookies_total_size', total_cookie_size)
log.debug(u'cookies_total_size = %d', total_cookie_size)

78
openedx/core/lib/tests/test_request_utils.py
View File

@@ -2,12 +2,20 @@
import unittest
from unittest.mock import Mock, patch, call
from django.conf import settings
from django.core.exceptions import SuspiciousOperation
from django.test.client import RequestFactory
from openedx.core.lib.request_utils import get_request_or_stub, course_id_from_url, safe_get_host
from edx_toggles.toggles.testutils import override_waffle_flag
from openedx.core.lib.request_utils import (
get_request_or_stub,
course_id_from_url,
safe_get_host,
CookieMonitoringMiddleware,
CAPTURE_COOKIE_SIZES,
)
class RequestUtilTestCase(unittest.TestCase):
@@ -83,3 +91,71 @@ class RequestUtilTestCase(unittest.TestCase):
self.assertEqual(course_id.org, org)
self.assertEqual(course_id.course, course)
self.assertEqual(course_id.run, run)
@patch("openedx.core.lib.request_utils.CAPTURE_COOKIE_SIZES")
@patch("openedx.core.lib.request_utils.set_custom_attribute")
def test_cookie_monitoring(self, mock_set_custom_attribute, mock_capture_cookie_sizes):
mock_capture_cookie_sizes.is_enabled.return_value = True
middleware = CookieMonitoringMiddleware()
mock_request = Mock()
mock_request.COOKIES = {
"a": "." * 100,
"_b": "." * 13,
"_c_": "." * 13,
"a.b": "." * 10,
"a.c": "." * 10,
"b.": "." * 13,
"b_a": "." * 15,
"b_c": "." * 15,
}
middleware.process_request(mock_request)
mock_set_custom_attribute.assert_has_calls([
call('cookies.a.size', 100),
call('cookies._b.size', 13),
call('cookies._c_.size', 13),
call('cookies.a.b.size', 10),
call('cookies.a.c.size', 10),
call('cookies.b..size', 13),
call('cookies.b_a.size', 15),
call('cookies.b_c.size', 15),
call('cookies.a.group.size', 20),
call('cookies.b.group.size', 43),
call('cookies.max.name', 'a'),
call('cookies.max.size', 100),
# Test that single cookie is also treated as a group for max calculations.
call('cookies.max.group.name', 'a'),
call('cookies.max.group.size', 100),
call('cookies_total_size', 189)
])
@patch("openedx.core.lib.request_utils.CAPTURE_COOKIE_SIZES")
@patch("openedx.core.lib.request_utils.set_custom_attribute")
def test_cookie_monitoring_max_group(self, mock_set_custom_attribute, mock_capture_cookie_sizes):
mock_capture_cookie_sizes.is_enabled.return_value = True
middleware = CookieMonitoringMiddleware()
mock_request = Mock()
mock_request.COOKIES = {
"a": "." * 10,
"b_a": "." * 15,
"b_c": "." * 20,
}
middleware.process_request(mock_request)
mock_set_custom_attribute.assert_has_calls([
call('cookies.a.size', 10),
call('cookies.b_a.size', 15),
call('cookies.b_c.size', 20),
call('cookies.b.group.size', 35),
call('cookies.max.name', 'b_c'),
call('cookies.max.size', 20),
call('cookies.max.group.name', 'b'),
call('cookies.max.group.size', 35),
call('cookies_total_size', 45)
])