Merge pull request #18807 from edx/robrap/ARCH-220-use-new-monitoring

ARCH-220: Migrate to edx-django-utils monitoring.
This commit is contained in:
Robert Raposa
2018-08-17 16:29:16 -04:00
committed by GitHub
21 changed files with 33 additions and 364 deletions

View File

@@ -472,12 +472,12 @@ XQUEUE_INTERFACE = {
MIDDLEWARE_CLASSES = [
'crum.CurrentRequestUserMiddleware',
# Clears request cache. This is a safer request cache.
'edx_django_utils.cache.middleware.RequestCacheMiddleware',
# Deprecated, but well entrenched RequestCache
'openedx.core.djangoapps.request_cache.middleware.RequestCache',
'openedx.core.djangoapps.monitoring_utils.middleware.MonitoringMemoryMiddleware',
# A newer and safer request cache.
'edx_django_utils.cache.middleware.RequestCacheMiddleware',
'edx_django_utils.monitoring.middleware.MonitoringMemoryMiddleware',
'openedx.core.djangoapps.header_control.middleware.HeaderControlMiddleware',
'django.middleware.cache.UpdateCacheMiddleware',

View File

@@ -15,7 +15,7 @@ from django.urls import reverse
from django.shortcuts import redirect
from django.utils.translation import ugettext as _
from django.views.decorators.csrf import ensure_csrf_cookie
from edx_django_utils import monitoring as monitoring_utils
from opaque_keys.edx.keys import CourseKey
from pytz import UTC
from six import text_type, iteritems
@@ -28,7 +28,6 @@ from edxmako.shortcuts import render_to_response, render_to_string
from entitlements.models import CourseEntitlement
from lms.djangoapps.commerce.utils import EcommerceService # pylint: disable=import-error
from lms.djangoapps.verify_student.services import IDVerificationService
from openedx.core.djangoapps import monitoring_utils
from openedx.core.djangoapps.catalog.utils import (
get_programs,
get_pseudo_session_for_entitlement,

View File

@@ -35,6 +35,7 @@ from django.utils.translation import get_language, ungettext
from django.utils.translation import ugettext as _
from django.views.decorators.csrf import csrf_exempt, ensure_csrf_cookie
from django.views.decorators.http import require_GET, require_POST
from edx_django_utils import monitoring as monitoring_utils
from eventtracking import tracker
from ipware.ip import get_ip
# Note that this lives in LMS, so this dependency should be refactored.
@@ -54,7 +55,6 @@ import track.views
from course_modes.models import CourseMode
from edxmako.shortcuts import render_to_response, render_to_string
from entitlements.models import CourseEntitlement
from openedx.core.djangoapps import monitoring_utils
from openedx.core.djangoapps.catalog.utils import (
get_programs_with_type,
)

View File

@@ -19,6 +19,7 @@ from django.urls import reverse
from django.http import Http404, HttpResponse, HttpResponseForbidden
from django.views.decorators.clickjacking import xframe_options_exempt
from django.views.decorators.csrf import csrf_exempt
from edx_django_utils.monitoring import set_custom_metrics_for_course_key, set_monitoring_transaction_name
from edx_proctoring.services import ProctoringService
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey, UsageKey
@@ -51,7 +52,6 @@ from lms.djangoapps.verify_student.services import XBlockVerificationService
from openedx.core.djangoapps.bookmarks.services import BookmarksService
from openedx.core.djangoapps.crawlers.models import CrawlersConfig
from openedx.core.djangoapps.credit.services import CreditService
from openedx.core.djangoapps.monitoring_utils import set_custom_metrics_for_course_key, set_monitoring_transaction_name
from openedx.core.djangoapps.util.user_utils import SystemUser
from openedx.core.lib.gating.services import GatingService
from openedx.core.lib.license import wrap_with_license

View File

@@ -13,12 +13,12 @@ from django.core.paginator import Paginator
from django.contrib.auth.models import User
from django.db import transaction
from django.db.utils import IntegrityError
from edx_django_utils import monitoring as monitoring_utils
from edx_user_state_client.interface import XBlockUserState, XBlockUserStateClient
from xblock.fields import Scope
import dogstats_wrapper as dog_stats_api
from courseware.models import BaseStudentModuleHistory, StudentModule
from openedx.core.djangoapps import monitoring_utils
try:
import simplejson as json

View File

@@ -19,6 +19,7 @@ from django.utils.translation import ugettext as _
from django.views.decorators.cache import cache_control
from django.views.decorators.csrf import ensure_csrf_cookie
from django.views.generic import View
from edx_django_utils.monitoring import set_custom_metrics_for_course_key
from opaque_keys.edx.keys import CourseKey
from web_fragments.fragment import Fragment
@@ -29,7 +30,6 @@ from lms.djangoapps.gating.api import get_entrance_exam_score_ratio, get_entranc
from lms.djangoapps.grades.course_grade_factory import CourseGradeFactory
from openedx.core.djangoapps.crawlers.models import CrawlersConfig
from openedx.core.djangoapps.lang_pref import LANGUAGE_KEY
from openedx.core.djangoapps.monitoring_utils import set_custom_metrics_for_course_key
from openedx.core.djangoapps.user_api.preferences.api import get_user_preference
from openedx.core.djangoapps.util.user_messages import PageLevelMessages
from openedx.core.djangoapps.waffle_utils import WaffleSwitchNamespace, WaffleFlagNamespace, CourseWaffleFlag

View File

@@ -27,6 +27,7 @@ from django.views.decorators.clickjacking import xframe_options_exempt
from django.views.decorators.csrf import ensure_csrf_cookie
from django.views.decorators.http import require_GET, require_http_methods, require_POST
from django.views.generic import View
from edx_django_utils.monitoring import set_custom_metrics_for_course_key
from eventtracking import tracker
from ipware.ip import get_ip
from markupsafe import escape
@@ -81,7 +82,6 @@ from openedx.core.djangoapps.credit.api import (
is_user_eligible_for_credit
)
from openedx.core.djangoapps.models.course_details import CourseDetails
from openedx.core.djangoapps.monitoring_utils import set_custom_metrics_for_course_key
from openedx.core.djangoapps.plugin_api.views import EdxFragmentView
from openedx.core.djangoapps.programs.utils import ProgramMarketingDataExtender
from openedx.core.djangoapps.self_paced.models import SelfPacedConfiguration

View File

@@ -17,6 +17,7 @@ from django.template.loader import render_to_string
from django.utils.translation import get_language_bidi
from django.views.decorators.csrf import ensure_csrf_cookie
from django.views.decorators.http import require_GET, require_http_methods
from edx_django_utils.monitoring import function_trace
from opaque_keys.edx.keys import CourseKey
from rest_framework import status
from web_fragments.fragment import Fragment
@@ -44,7 +45,6 @@ from django_comment_client.utils import (
from django_comment_common.models import CourseDiscussionSettings
from django_comment_common.utils import ThreadContext, get_course_discussion_settings, set_course_discussion_settings
from openedx.core.djangoapps.plugin_api.views import EdxFragmentView
from openedx.core.djangoapps.monitoring_utils import function_trace
from student.models import CourseEnrollment
from util.json_request import JsonResponse, expect_json
from xmodule.modulestore.django import modulestore

View File

@@ -12,11 +12,12 @@ from django.conf import settings
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.db.utils import DatabaseError
from edx_django_utils.monitoring import set_custom_metric, set_custom_metrics_for_course_key
from lms.djangoapps.course_blocks.api import get_course_blocks
from lms.djangoapps.grades.config.models import ComputeGradesSetting
from opaque_keys.edx.keys import CourseKey, UsageKey
from opaque_keys.edx.locator import CourseLocator
from openedx.core.djangoapps.monitoring_utils import set_custom_metric, set_custom_metrics_for_course_key
from student.models import CourseEnrollment
from submissions import api as sub_api
from track.event_transaction_utils import set_event_transaction_id, set_event_transaction_type

View File

@@ -1216,11 +1216,12 @@ CREDIT_NOTIFICATION_CACHE_TIMEOUT = 5 * 60 * 60
MIDDLEWARE_CLASSES = [
'crum.CurrentRequestUserMiddleware',
# Clears request cache. This is a safer request cache.
'edx_django_utils.cache.middleware.RequestCacheMiddleware',
# Deprecated, but well entrenched RequestCache
'openedx.core.djangoapps.request_cache.middleware.RequestCache',
'openedx.core.djangoapps.monitoring_utils.middleware.MonitoringCustomMetrics',
# A newer and safer request cache.
'edx_django_utils.cache.middleware.RequestCacheMiddleware',
'edx_django_utils.monitoring.middleware.MonitoringCustomMetricsMiddleware',
'mobile_api.middleware.AppVersionUpgrade',
'openedx.core.djangoapps.header_control.middleware.HeaderControlMiddleware',

View File

@@ -1,112 +0,0 @@
"""
This is an interface to the monitoring_utils middleware. Functions
defined in this module can be used to report monitoring custom metrics.
Usage:
from openedx.core.djangoapps import monitoring_utils
...
monitoring_utils.accumulate('xb_user_state.get_many.num_items', 4)
There is no need to do anything else. The metrics are automatically cleared
before the next request.
We try to keep track of our custom metrics at:
https://openedx.atlassian.net/wiki/display/PERF/Custom+Metrics+in+New+Relic
At this time, these custom metrics will only be reported to New Relic.
TODO: supply additional public functions for storing strings and booleans.
"""
from contextlib import contextmanager
from . import middleware
try:
import newrelic.agent
except ImportError:
newrelic = None # pylint: disable=invalid-name
def accumulate(name, value):
"""
Accumulate monitoring custom metric for the current request.
The named metric is accumulated by a numerical amount using the sum. All
metrics are queued up in the request_cache for this request. At the end of
the request, the monitoring_utils middleware will batch report all
queued accumulated metrics to the monitoring tool (e.g. New Relic).
Arguments:
name (str): The metric name. It should be period-delimited, and
increase in specificity from left to right. For example:
'xb_user_state.get_many.num_items'.
value (number): The amount to accumulate into the named metric. When
accumulate() is called multiple times for a given metric name
during a request, the sum of the values for each call is reported
for that metric. For metrics which don't make sense to accumulate,
make sure to only call this function once during a request.
"""
middleware.MonitoringCustomMetrics.accumulate_metric(name, value)
def increment(name):
"""
Increment a monitoring custom metric representing a counter.
Here we simply accumulate a new custom metric with a value of 1, and the
middleware should automatically aggregate this metric.
"""
accumulate(name, 1)
def set_custom_metrics_for_course_key(course_key):
"""
Set monitoring custom metrics related to a course key.
This is not cached, and only support reporting to New Relic Insights.
"""
if not newrelic:
return
newrelic.agent.add_custom_parameter('course_id', unicode(course_key))
newrelic.agent.add_custom_parameter('org', unicode(course_key.org))
def set_custom_metric(key, value):
"""
Set monitoring custom metric.
This is not cached, and only support reporting to New Relic Insights.
"""
if not newrelic:
return
newrelic.agent.add_custom_parameter(key, value)
def set_monitoring_transaction_name(name, group=None, priority=None):
"""
Sets the transaction name for monitoring.
This is not cached, and only support reporting to New Relic.
"""
if not newrelic:
return
newrelic.agent.set_transaction_name(name, group, priority)
@contextmanager
def function_trace(function_name):
"""
Wraps a chunk of code that we want to appear as a separate, explicit,
segment in our monitoring tools.
"""
if newrelic:
nr_transaction = newrelic.agent.current_transaction()
with newrelic.agent.FunctionTrace(nr_transaction, function_name):
yield
else:
yield

View File

@@ -1,175 +0,0 @@
"""
Middleware for handling the storage, aggregation, and reporting of custom
metrics for monitoring.
At this time, the custom metrics can only be reported to New Relic.
This middleware will only call on the newrelic agent if there are any metrics
to report for this request, so it will not incur any processing overhead for
request handlers which do not record custom metrics.
"""
import logging
from uuid import uuid4
import psutil
from openedx.core.djangoapps.request_cache import get_cache
from openedx.core.djangoapps.waffle_utils import WaffleSwitchNamespace
log = logging.getLogger(__name__)
try:
import newrelic.agent
except ImportError:
log.warning("Unable to load NewRelic agent module")
newrelic = None # pylint: disable=invalid-name
REQUEST_CACHE_KEY = 'monitoring_custom_metrics'
WAFFLE_NAMESPACE = 'monitoring_utils'
class MonitoringCustomMetrics(object):
"""
The middleware class. Make sure to add below the request cache in
MIDDLEWARE_CLASSES.
"""
@classmethod
def _get_metrics_cache(cls):
"""
Get a reference to the part of the request cache wherein we store New
Relic custom metrics related to the current request.
"""
return get_cache(name=REQUEST_CACHE_KEY)
@classmethod
def accumulate_metric(cls, name, value):
"""
Accumulate a custom metric (name and value) in the metrics cache.
"""
metrics_cache = cls._get_metrics_cache()
metrics_cache.setdefault(name, 0)
metrics_cache[name] += value
@classmethod
def _batch_report(cls):
"""
Report the collected custom metrics to New Relic.
"""
if not newrelic:
return
metrics_cache = cls._get_metrics_cache()
for metric_name, metric_value in metrics_cache.iteritems():
newrelic.agent.add_custom_parameter(metric_name, metric_value)
# Whether or not there was an exception, report any custom NR metrics that
# may have been collected.
def process_response(self, request, response): # pylint: disable=unused-argument
"""
Django middleware handler to process a response
"""
self._batch_report()
return response
def process_exception(self, request, exception): # pylint: disable=unused-argument
"""
Django middleware handler to process an exception
"""
self._batch_report()
return None
class MonitoringMemoryMiddleware(object):
"""
Middleware for monitoring memory usage.
"""
memory_data_key = u'memory_data'
guid_key = u'guid_key'
def process_request(self, request):
if self._is_enabled():
self._cache[self.guid_key] = unicode(uuid4())
log_prefix = self._log_prefix(u"Before", request)
self._cache[self.memory_data_key] = self._memory_data(log_prefix)
def process_response(self, request, response):
if self._is_enabled():
log_prefix = self._log_prefix(u"After", request)
new_memory_data = self._memory_data(log_prefix)
log_prefix = self._log_prefix(u"Diff", request)
self._log_diff_memory_data(log_prefix, new_memory_data, self._cache.get(self.memory_data_key))
return response
@property
def _cache(self):
"""
Namespaced request cache for tracking memory usage.
"""
return get_cache(name='monitoring_memory')
def _log_prefix(self, prefix, request):
"""
Returns a formatted prefix for logging for the given request.
"""
# After a celery task runs, the request cache is cleared. So if celery
# tasks are running synchronously (CELERY_ALWAYS _EAGER), "guid_key"
# will no longer be in the request cache when process_response executes.
cached_guid = self._cache.get(self.guid_key) or u"without_guid"
return u"{} request '{} {} {}'".format(prefix, request.method, request.path, cached_guid)
def _memory_data(self, log_prefix):
"""
Returns a dict with information for current memory utilization.
Uses log_prefix in log statements.
"""
machine_data = psutil.virtual_memory()
process = psutil.Process()
process_data = {
'memory_info': process.get_memory_info(),
'ext_memory_info': process.get_ext_memory_info(),
'memory_percent': process.get_memory_percent(),
'cpu_percent': process.get_cpu_percent(),
}
log.info(u"%s Machine memory usage: %s; Process memory usage: %s", log_prefix, machine_data, process_data)
return {
'machine_data': machine_data,
'process_data': process_data,
}
def _log_diff_memory_data(self, prefix, new_memory_data, old_memory_data):
"""
Computes and logs the difference in memory utilization
between the given old and new memory data.
"""
def _vmem_used(memory_data):
return memory_data['machine_data'].used
def _process_mem_percent(memory_data):
return memory_data['process_data']['memory_percent']
def _process_rss(memory_data):
return memory_data['process_data']['memory_info'].rss
def _process_vms(memory_data):
return memory_data['process_data']['memory_info'].vms
if new_memory_data and old_memory_data:
log.info(
u"%s Diff Vmem used: %s, Diff percent memory: %s, Diff rss: %s, Diff vms: %s",
prefix,
_vmem_used(new_memory_data) - _vmem_used(old_memory_data),
_process_mem_percent(new_memory_data) - _process_mem_percent(old_memory_data),
_process_rss(new_memory_data) - _process_rss(old_memory_data),
_process_vms(new_memory_data) - _process_vms(old_memory_data),
)
def _is_enabled(self):
"""
Returns whether this middleware is enabled.
"""
return WaffleSwitchNamespace(name=WAFFLE_NAMESPACE).is_enabled(u'enable_memory_middleware')

View File

@@ -1,47 +0,0 @@
"""
Tests for monitoring custom metrics.
"""
from django.test import TestCase
from mock import call, patch
from openedx.core.djangoapps import monitoring_utils
from openedx.core.djangoapps.monitoring_utils.middleware import MonitoringCustomMetrics
class TestMonitoringCustomMetrics(TestCase):
"""
Test the monitoring_utils middleware and helpers
"""
@patch('newrelic.agent')
def test_custom_metrics_with_new_relic(self, mock_newrelic_agent):
"""
Test normal usage of collecting custom metrics and reporting to New Relic
"""
monitoring_utils.accumulate('hello', 10)
monitoring_utils.accumulate('world', 10)
monitoring_utils.accumulate('world', 10)
monitoring_utils.increment('foo')
monitoring_utils.increment('foo')
# based on the metric data above, we expect the following calls to newrelic:
nr_agent_calls_expected = [
call('hello', 10),
call('world', 20),
call('foo', 2),
]
# fake a response to trigger metrics reporting
MonitoringCustomMetrics().process_response(
'fake request',
'fake response',
)
# Assert call counts to newrelic.agent.add_custom_parameter()
expected_call_count = len(nr_agent_calls_expected)
measured_call_count = mock_newrelic_agent.add_custom_parameter.call_count
self.assertEqual(expected_call_count, measured_call_count)
# Assert call args to newrelic.agent.add_custom_parameter(). Due to
# the nature of python dicts, call order is undefined.
mock_newrelic_agent.add_custom_parameter.has_calls(nr_agent_calls_expected, any_order=True)

View File

@@ -235,7 +235,7 @@ class TestAccessTokenView(AccessTokenLoginMixin, mixins.AccessTokenMixin, _Dispa
(None, 'no_token_type_supplied'),
)
@ddt.unpack
@patch('openedx.core.djangoapps.monitoring_utils.set_custom_metric')
@patch('edx_django_utils.monitoring.set_custom_metric')
def test_access_token_metrics(self, token_type, expected_token_type, mock_set_custom_metric):
response = self._post_request(self.user, self.dot_app, token_type=token_type)
self.assertEqual(response.status_code, 200)
@@ -245,7 +245,7 @@ class TestAccessTokenView(AccessTokenLoginMixin, mixins.AccessTokenMixin, _Dispa
]
mock_set_custom_metric.assert_has_calls(expected_calls, any_order=True)
@patch('openedx.core.djangoapps.monitoring_utils.set_custom_metric')
@patch('edx_django_utils.monitoring.set_custom_metric')
def test_access_token_metrics_for_bad_request(self, mock_set_custom_metric):
grant_type = dot_models.Application.GRANT_PASSWORD
invalid_body = {
@@ -596,13 +596,13 @@ class TestViewDispatch(TestCase):
]
mock_set_custom_metric.assert_has_calls(expected_calls, any_order=True)
@patch('openedx.core.djangoapps.monitoring_utils.set_custom_metric')
@patch('edx_django_utils.monitoring.set_custom_metric')
def test_dispatching_post_to_dot(self, mock_set_custom_metric):
request = self._post_request('dot-id')
self.assertEqual(self.view.select_backend(request), self.dot_adapter.backend)
self._verify_oauth_metrics_calls(mock_set_custom_metric, 'dot')
@patch('openedx.core.djangoapps.monitoring_utils.set_custom_metric')
@patch('edx_django_utils.monitoring.set_custom_metric')
def test_dispatching_post_to_dop(self, mock_set_custom_metric):
request = self._post_request('dop-id')
self.assertEqual(self.view.select_backend(request), self.dop_adapter.backend)

View File

@@ -10,13 +10,13 @@ import logging
from django.conf import settings
from django.views.generic import View
from edx_django_utils import monitoring as monitoring_utils
from edx_oauth2_provider import views as dop_views # django-oauth2-provider views
from oauth2_provider import models as dot_models # django-oauth-toolkit
from oauth2_provider import views as dot_views
from ratelimit import ALL
from ratelimit.mixins import RatelimitMixin
from openedx.core.djangoapps import monitoring_utils
from openedx.core.djangoapps.auth_exchange import views as auth_exchange_views
from openedx.core.lib.token_utils import JwtBuilder

View File

@@ -8,12 +8,11 @@ from django.contrib.auth.models import User
from django.contrib.staticfiles.templatetags.staticfiles import static
from django.urls import reverse
from django.db.models import F, Q
from edx_ace.recipient_resolver import RecipientResolver
from edx_ace.recipient import Recipient
from edx_django_utils.monitoring import function_trace, set_custom_metric
from courseware.date_summary import verified_upgrade_deadline_link, verified_upgrade_link_is_valid
from openedx.core.djangoapps.monitoring_utils import function_trace, set_custom_metric
from openedx.core.djangoapps.schedules.content_highlights import get_week_highlights
from openedx.core.djangoapps.schedules.exceptions import CourseUpdateDoesNotExist
from openedx.core.djangoapps.schedules.models import Schedule, ScheduleExperience

View File

@@ -15,9 +15,9 @@ from celery_utils.persist_on_failure import LoggedPersistOnFailureTask
from edx_ace import ace
from edx_ace.message import Message
from edx_ace.utils.date import deserialize, serialize
from edx_django_utils.monitoring import set_custom_metric
from opaque_keys.edx.keys import CourseKey
from openedx.core.djangoapps.monitoring_utils import set_custom_metric
from openedx.core.djangoapps.schedules import message_types
from openedx.core.djangoapps.schedules.models import Schedule, ScheduleConfig
from openedx.core.djangoapps.schedules import resolvers

View File

@@ -19,7 +19,9 @@ from django.core.cache import caches
from django.db import DEFAULT_DB_ALIAS, connections
from django.test import RequestFactory, TestCase, override_settings
from django.test.utils import CaptureQueriesContext
from openedx.core.djangoapps.request_cache.middleware import RequestCache
from edx_django_utils.cache import RequestCache
from openedx.core.djangoapps.request_cache.middleware import RequestCache as DeprecatedRequestCache
class CacheIsolationMixin(object):
@@ -118,7 +120,8 @@ class CacheIsolationMixin(object):
# Clear that.
sites.models.SITE_CACHE.clear()
RequestCache.clear_request_cache()
DeprecatedRequestCache.clear_request_cache()
RequestCache.clear_all_namespaces()
class CacheIsolationTestCase(CacheIsolationMixin, TestCase):

View File

@@ -115,7 +115,7 @@ edx-completion==0.1.8
edx-django-oauth2-provider==1.3.4
edx-django-release-util==0.3.1
edx-django-sites-extensions==2.3.1
edx-django-utils==0.4.1
edx-django-utils==0.5.1
edx-drf-extensions==1.6.1
edx-enterprise==0.72.5
edx-i18n-tools==0.4.6

View File

@@ -135,7 +135,7 @@ edx-completion==0.1.8
edx-django-oauth2-provider==1.3.4
edx-django-release-util==0.3.1
edx-django-sites-extensions==2.3.1
edx-django-utils==0.4.1
edx-django-utils==0.5.1
edx-drf-extensions==1.6.1
edx-enterprise==0.72.5
edx-i18n-tools==0.4.6

View File

@@ -130,7 +130,7 @@ edx-completion==0.1.8
edx-django-oauth2-provider==1.3.4
edx-django-release-util==0.3.1
edx-django-sites-extensions==2.3.1
edx-django-utils==0.4.1
edx-django-utils==0.5.1
edx-drf-extensions==1.6.1
edx-enterprise==0.72.5
edx-i18n-tools==0.4.6