Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #26065 from edx/feanil/update_cookie_metrics

Browse Source 
Add more cookie logging code.
This commit is contained in:
Feanil Patel
2021-01-14 16:30:34 -05:00
committed by GitHub
parent b4461ecc05 e5e2b7f7b6
commit 4047a72701
2 changed files with 176 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
openedx/core/lib/request_utils.py
View File

@@ -102,18 +102,104 @@ class CookieMonitoringMiddleware(MiddlewareMixin):
Don't log contents of cookies because that might cause a security issue.
We just want to see if any cookies are growing out of control.
A useful NRQL Query:
SELECT count(*), max(`cookies.max.group.size`) from Transaction FACET
`cookies.max.group.name`
SELECT * FROM Transaction WHERE cookies_total_size > 6000
Attributes that are added by this middleware:
cookies.<N>.name: The name of the Nth largest cookie
cookies.<N>.size: The size of the Nth largest cookie
cookies..group.<N>.name: The name of the Nth largest cookie.
cookies.group.<N>.size: The size of the Nth largest cookie group.
cookies.max.name: The name of the largest cookie sent by the user.
cookies.max.size: The size of the largest cookie sent by the user.
cookies.max.group.name: The name of the largest group of cookies. A single cookie
counts as a group of one for this calculation.
cookies.max.group.size: The sum total size of all the cookies in the largest group.
cookies_total_size: The sum total size of all cookies in this request.
Related Settings:
- `request_utils.capture_cookie_sizes` is the waffle flag that control whether this
middleware logs anything or not.
- TOP_N_COOKIES_CAPTURED(Default: 5) controls how many cookies to log.
- TOP_N_COOKIE_GROUPS_CAPTURED(Default: 5): controls how many cookie groups to capture.
"""
if not CAPTURE_COOKIE_SIZES.is_enabled():
return
cookie_names_to_size = {
name: len(value)
for name, value in request.COOKIES.items()
}
for name, size in cookie_names_to_size.items():
attribute_name = 'cookies.{}.size'.format(name)
set_custom_attribute(attribute_name, size)
log.debug(u'%s = %d', attribute_name, size)
# Capture the N largest cookies
top_n_cookies_captured = getattr(settings, "TOP_N_COOKIES_CAPTURED", 5)
top_n_cookie_groups_captured = getattr(settings, "TOP_N_COOKIE_GROUPS_CAPTURED", 5)
cookie_names_to_size = {}
cookie_groups_to_size = {}
for name, value in request.COOKIES.items():
# Get cookie size for all cookies.
cookie_size = len(value)
cookie_names_to_size[name] = cookie_size
# Group cookies by their prefix seperated by a period or underscore
grouping_name = re.split('[._]', name, 1)[0]
if grouping_name and grouping_name != name:
# Add or update the size for this group.
cookie_groups_to_size[grouping_name] = cookie_groups_to_size.get(grouping_name, 0) + cookie_size
max_cookie_name = max(cookie_names_to_size, key=lambda name: cookie_names_to_size[name])
max_cookie_size = cookie_names_to_size[max_cookie_name]
max_group_cookie_name = max(cookie_groups_to_size, key=lambda name: cookie_groups_to_size[name])
max_group_cookie_size = cookie_groups_to_size[max_group_cookie_name]
# If a single cookies is bigger than any group of cookies, we want max_group... to reflect that.
# Treating an individual cookie as a group of 1 for calculating the max.
if max_group_cookie_size < max_cookie_size:
max_group_cookie_name = max_cookie_name
max_group_cookie_size = max_cookie_size
# Log only the top N biggest cookies.
top_n_cookies = sorted(
cookie_names_to_size,
key=lambda x: cookie_names_to_size[x],
reverse=True,
)[:top_n_cookies_captured]
for index, name in enumerate(top_n_cookies, start=1):
size = cookie_names_to_size[name]
name_attribute = 'cookies.{}.name'.format(index)
size_attribute = 'cookies.{}.size'.format(index)
set_custom_attribute(name_attribute, name)
set_custom_attribute(size_attribute, size)
log.debug(u'%s = %d', name, size)
# Log only the top N biggest groups.
top_n_cookie_groups = sorted(
cookie_groups_to_size,
key=lambda x: cookie_groups_to_size[x],
reverse=True,
)[:top_n_cookie_groups_captured]
for index, name in enumerate(top_n_cookie_groups, start=1):
size = cookie_groups_to_size[name]
name_attribute = 'cookies.group.{}.name'.format(index)
size_attribute = 'cookies.group.{}.size'.format(index)
set_custom_attribute(name_attribute, name)
set_custom_attribute(size_attribute, size)
log.debug(u'%s = %d', name, size)
set_custom_attribute('cookies.max.name', max_cookie_name)
set_custom_attribute('cookies.max.size', max_cookie_size)
set_custom_attribute('cookies.max.group.name', max_group_cookie_name)
set_custom_attribute('cookies.max.group.size', max_group_cookie_size)
total_cookie_size = sum(cookie_names_to_size.values())
set_custom_attribute('cookies_total_size', total_cookie_size)

83
openedx/core/lib/tests/test_request_utils.py
View File

@@ -2,12 +2,18 @@
import unittest
from unittest.mock import Mock, patch, call
from django.conf import settings
from django.core.exceptions import SuspiciousOperation
from django.test.client import RequestFactory
from openedx.core.lib.request_utils import get_request_or_stub, course_id_from_url, safe_get_host
from openedx.core.lib.request_utils import (
get_request_or_stub,
course_id_from_url,
safe_get_host,
CookieMonitoringMiddleware,
)
class RequestUtilTestCase(unittest.TestCase):
@@ -83,3 +89,78 @@ class RequestUtilTestCase(unittest.TestCase):
self.assertEqual(course_id.org, org)
self.assertEqual(course_id.course, course)
self.assertEqual(course_id.run, run)
@patch("openedx.core.lib.request_utils.CAPTURE_COOKIE_SIZES")
@patch("openedx.core.lib.request_utils.set_custom_attribute")
def test_cookie_monitoring(self, mock_set_custom_attribute, mock_capture_cookie_sizes):
mock_capture_cookie_sizes.is_enabled.return_value = True
middleware = CookieMonitoringMiddleware()
mock_request = Mock()
mock_request.COOKIES = {
"a": "." * 100,
"_b": "." * 13,
"_c_": "." * 13,
"a.b": "." * 10,
"a.c": "." * 10,
"b.": "." * 13,
"b_a": "." * 15,
"b_c": "." * 15,
}
middleware.process_request(mock_request)
mock_set_custom_attribute.assert_has_calls([
call('cookies.1.name', 'a'),
call('cookies.1.size', 100),
call('cookies.2.name', 'b_a'),
call('cookies.2.size', 15),
call('cookies.3.name', 'b_c'),
call('cookies.3.size', 15),
call('cookies.4.name', '_b'),
call('cookies.4.size', 13),
call('cookies.5.name', '_c_'),
call('cookies.5.size', 13),
call('cookies.group.1.name', 'b'),
call('cookies.group.1.size', 43),
call('cookies.group.2.name', 'a'),
call('cookies.group.2.size', 20),
call('cookies.max.name', 'a'),
call('cookies.max.size', 100),
call('cookies.max.group.name', 'a'),
call('cookies.max.group.size', 100),
call('cookies_total_size', 189),
])
@patch("openedx.core.lib.request_utils.CAPTURE_COOKIE_SIZES")
@patch("openedx.core.lib.request_utils.set_custom_attribute")
def test_cookie_monitoring_max_group(self, mock_set_custom_attribute, mock_capture_cookie_sizes):
mock_capture_cookie_sizes.is_enabled.return_value = True
middleware = CookieMonitoringMiddleware()
mock_request = Mock()
mock_request.COOKIES = {
"a": "." * 10,
"b_a": "." * 15,
"b_c": "." * 20,
}
middleware.process_request(mock_request)
mock_set_custom_attribute.assert_has_calls([
call('cookies.1.name', 'b_c'),
call('cookies.1.size', 20),
call('cookies.2.name', 'b_a'),
call('cookies.2.size', 15),
call('cookies.3.name', 'a'),
call('cookies.3.size', 10),
call('cookies.group.1.name', 'b'),
call('cookies.group.1.size', 35),
call('cookies.max.name', 'b_c'),
call('cookies.max.size', 20),
call('cookies.max.group.name', 'b'),
call('cookies.max.group.size', 35),
call('cookies_total_size', 45)
])