Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove all references to datadog from our code.

Browse Source
This commit is contained in:
Diana Huang
2018-12-11 13:37:19 -05:00
parent 577315f2a7
commit 6572d99e76
44 changed files with 114 additions and 614 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
lms/djangoapps/bulk_email/tasks.py
View File

@@ -38,7 +38,6 @@ from django.utils.translation import ugettext as _
from markupsafe import escape
from six import text_type
import dogstats_wrapper as dog_stats_api
from bulk_email.models import CourseEmail, Optout
from courseware.courses import get_course
from lms.djangoapps.instructor_task.models import InstructorTask
@@ -295,14 +294,13 @@ def send_course_email(entry_id, email_id, to_list, global_email_context, subtask
new_subtask_status = None
try:
course_title = global_email_context['course_title']
with dog_stats_api.timer('course_email.single_task.time.overall', tags=[_statsd_tag(course_title)]):
new_subtask_status, send_exception = _send_course_email(
entry_id,
email_id,
to_list,
global_email_context,
subtask_status,
)
new_subtask_status, send_exception = _send_course_email(
entry_id,
email_id,
to_list,
global_email_context,
subtask_status,
)
except Exception:
# Unexpected exception. Try to write out the failure to the entry before failing.
log.exception("Send-email task %s for email %s: failed unexpectedly!", current_task_id, email_id)
@@ -386,7 +384,7 @@ def _get_source_address(course_id, course_title, course_language, truncate=True)
from_addr_format = u'{name} {email}'.format(
# Translators: Bulk email from address e.g. ("Physics 101" Course Staff)
name=_('"{course_title}" Course Staff'),
email=u'<{course_name}-{from_email}>',
email=u'<{course_name}-{from_email}>', # xss-lint: disable=python-wrap-html
)
def format_address(course_title_no_quotes):
@@ -566,8 +564,7 @@ def _send_course_email(entry_id, email_id, to_list, global_email_context, subtas
current_recipient['profile__name'],
email
)
with dog_stats_api.timer('course_email.single_send.time.overall', tags=[_statsd_tag(course_title)]):
connection.send_messages([email_msg])
connection.send_messages([email_msg])
except SMTPDataError as exc:
# According to SMTP spec, we'll retry error codes in the 4xx range. 5xx range indicates hard failure.
@@ -598,7 +595,6 @@ def _send_course_email(entry_id, email_id, to_list, global_email_context, subtas
email,
exc.smtp_error
)
dog_stats_api.increment('course_email.error', tags=[_statsd_tag(course_title)])
subtask_status.increment(failed=1)
except SINGLE_EMAIL_FAILURE_ERRORS as exc:
@@ -615,7 +611,6 @@ def _send_course_email(entry_id, email_id, to_list, global_email_context, subtas
email,
exc
)
dog_stats_api.increment('course_email.error', tags=[_statsd_tag(course_title)])
subtask_status.increment(failed=1)
else:
@@ -630,7 +625,6 @@ def _send_course_email(entry_id, email_id, to_list, global_email_context, subtas
total_recipients,
email
)
dog_stats_api.increment('course_email.sent', tags=[_statsd_tag(course_title)])
if settings.BULK_EMAIL_LOG_SENT_EMAILS:
log.info('Email with id %s sent to %s', email_id, email)
else:
@@ -667,7 +661,6 @@ def _send_course_email(entry_id, email_id, to_list, global_email_context, subtas
)
except INFINITE_RETRY_ERRORS as exc:
dog_stats_api.increment('course_email.infinite_retry', tags=[_statsd_tag(course_title)])
# Increment the "retried_nomax" counter, update other counters with progress to date,
# and set the state to RETRY:
subtask_status.increment(retried_nomax=1, state=RETRY)
@@ -679,7 +672,6 @@ def _send_course_email(entry_id, email_id, to_list, global_email_context, subtas
# Errors caught here cause the email to be retried. The entire task is actually retried
# without popping the current recipient off of the existing list.
# Errors caught are those that indicate a temporary condition that might succeed on retry.
dog_stats_api.increment('course_email.limited_retry', tags=[_statsd_tag(course_title)])
# Increment the "retried_withmax" counter, update other counters with progress to date,
# and set the state to RETRY:
subtask_status.increment(retried_withmax=1, state=RETRY)
@@ -688,7 +680,6 @@ def _send_course_email(entry_id, email_id, to_list, global_email_context, subtas
)
except BULK_EMAIL_FAILURE_ERRORS as exc:
dog_stats_api.increment('course_email.error', tags=[_statsd_tag(course_title)])
num_pending = len(to_list)
log.exception(('Task %s: email with id %d caused send_course_email task to fail '
'with "fatal" exception. %d emails unsent.'),
@@ -703,7 +694,6 @@ def _send_course_email(entry_id, email_id, to_list, global_email_context, subtas
# without popping the current recipient off of the existing list.
# These are unexpected errors. Since they might be due to a temporary condition that might
# succeed on retry, we give them a retry.
dog_stats_api.increment('course_email.limited_retry', tags=[_statsd_tag(course_title)])
log.exception(('Task %s: email with id %d caused send_course_email task to fail '
'with unexpected exception. Generating retry.'),
task_id, email_id)
@@ -838,11 +828,3 @@ def _submit_for_retry(entry_id, email_id, to_list, global_email_context,
num_failed = len(to_list)
subtask_status.increment(failed=num_failed, state=FAILURE)
return subtask_status, retry_exc
def _statsd_tag(course_title):
"""
Prefix the tag we will use for DataDog.
The tag also gets modified by our dogstats_wrapper code.
"""
return u"course_email:{0}".format(course_title)

6
lms/djangoapps/certificates/views/xqueue.py
View File

@@ -11,7 +11,6 @@ from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST
from opaque_keys.edx.keys import CourseKey
import dogstats_wrapper as dog_stats_api
from capa.xqueue_interface import XQUEUE_METRIC_NAME
from lms.djangoapps.certificates.api import generate_user_certificates
from lms.djangoapps.certificates.models import (
@@ -128,11 +127,6 @@ def update_certificate(request):
content_type='application/json'
)
dog_stats_api.increment(XQUEUE_METRIC_NAME, tags=[
u'action:update_certificate',
u'course_id:{}'.format(cert.course_id)
])
cert.save()
return HttpResponse(json.dumps({'return_code': 0}),
content_type='application/json')

48
lms/djangoapps/courseware/user_state_client.py
View File

@@ -17,7 +17,6 @@ from edx_django_utils import monitoring as monitoring_utils
from edx_user_state_client.interface import XBlockUserState, XBlockUserStateClient
from xblock.fields import Scope
import dogstats_wrapper as dog_stats_api
from courseware.models import BaseStudentModuleHistory, StudentModule
try:
@@ -105,27 +104,6 @@ class DjangoXBlockUserStateClient(XBlockUserStateClient):
usage_key = student_module.module_state_key.map_into_course(student_module.course_id)
yield (student_module, usage_key)
def _ddog_increment(self, evt_time, evt_name):
"""
DataDog increment method.
"""
dog_stats_api.increment(
'DjangoXBlockUserStateClient.{}'.format(evt_name),
timestamp=evt_time,
sample_rate=self.API_DATADOG_SAMPLE_RATE,
)
def _ddog_histogram(self, evt_time, evt_name, value):
"""
DataDog histogram method.
"""
dog_stats_api.histogram(
'DjangoXBlockUserStateClient.{}'.format(evt_name),
value,
timestamp=evt_time,
sample_rate=self.API_DATADOG_SAMPLE_RATE,
)
def _nr_metric_name(self, function_name, stat_name, block_type=None):
"""
Return a metric name (string) representing the provided descriptors.
@@ -195,22 +173,16 @@ class DjangoXBlockUserStateClient(XBlockUserStateClient):
self._nr_stat_increment('get_many', 'calls')
# keep track of blocks requested
self._ddog_histogram(evt_time, 'get_many.blks_requested', len(block_keys))
self._nr_stat_accumulate('get_many', 'blocks_requested', len(block_keys))
modules = self._get_student_modules(username, block_keys)
for module, usage_key in modules:
if module.state is None:
self._ddog_increment(evt_time, 'get_many.empty_state')
continue
state = json.loads(module.state)
state_length = len(module.state)
# record this metric before the check for empty state, so that we
# have some visibility into empty blocks.
self._ddog_histogram(evt_time, 'get_many.block_size', state_length)
# If the state is the empty dict, then it has been deleted, and so
# conformant UserStateClients should treat it as if it doesn't exist.
if state == {}:
@@ -233,9 +205,6 @@ class DjangoXBlockUserStateClient(XBlockUserStateClient):
# The rest of this method exists only to report metrics.
finish_time = time()
duration = (finish_time - evt_time) * 1000 # milliseconds
self._ddog_histogram(evt_time, 'get_many.blks_out', total_block_count)
self._ddog_histogram(evt_time, 'get_many.response_time', duration)
self._nr_stat_accumulate('get_many', 'duration', duration)
def set_many(self, username, block_keys_to_state, scope=Scope.user_state):
@@ -325,28 +294,19 @@ class DjangoXBlockUserStateClient(XBlockUserStateClient):
# Record whether a state row has been created or updated.
if created:
self._ddog_increment(evt_time, 'set_many.state_created')
self._nr_block_stat_increment('set_many', usage_key.block_type, 'blocks_created')
else:
self._ddog_increment(evt_time, 'set_many.state_updated')
self._nr_block_stat_increment('set_many', usage_key.block_type, 'blocks_updated')
# Event to record number of fields sent in to set/set_many.
self._ddog_histogram(evt_time, 'set_many.fields_in', len(state))
# Event to record number of new fields set in set/set_many.
num_new_fields_set = num_fields_after - num_fields_before
self._ddog_histogram(evt_time, 'set_many.fields_set', num_new_fields_set)
# Event to record number of existing fields updated in set/set_many.
num_fields_updated = max(0, len(state) - num_new_fields_set)
self._ddog_histogram(evt_time, 'set_many.fields_updated', num_fields_updated)
# Events for the entire set_many call.
finish_time = time()
duration = (finish_time - evt_time) * 1000 # milliseconds
self._ddog_histogram(evt_time, 'set_many.blks_updated', len(block_keys_to_state))
self._ddog_histogram(evt_time, 'set_many.response_time', duration)
self._nr_stat_accumulate('set_many', 'duration', duration)
def delete_many(self, username, block_keys, scope=Scope.user_state, fields=None):
@@ -363,13 +323,6 @@ class DjangoXBlockUserStateClient(XBlockUserStateClient):
raise ValueError("Only Scope.user_state is supported")
evt_time = time()
if fields is None:
self._ddog_increment(evt_time, 'delete_many.empty_state')
else:
self._ddog_histogram(evt_time, 'delete_many.field_count', len(fields))
self._ddog_histogram(evt_time, 'delete_many.block_count', len(block_keys))
student_modules = self._get_student_modules(username, block_keys)
for student_module, _ in student_modules:
if fields is None:
@@ -387,7 +340,6 @@ class DjangoXBlockUserStateClient(XBlockUserStateClient):
# Event for the entire delete_many call.
finish_time = time()
self._ddog_histogram(evt_time, 'delete_many.response_time', (finish_time - evt_time) * 1000)
def get_history(self, username, block_key, scope=Scope.user_state):
"""

4
lms/djangoapps/grades/course_grade_factory.py
View File

@@ -4,7 +4,6 @@ Course Grade Factory Class
from collections import namedtuple
from logging import getLogger
import dogstats_wrapper as dog_stats_api
from six import text_type
from openedx.core.djangoapps.signals.signals import COURSE_GRADE_CHANGED, COURSE_GRADE_NOW_PASSED
@@ -103,8 +102,7 @@ class CourseGradeFactory(object):
)
stats_tags = [u'action:{}'.format(course_data.course_key)]
for user in users:
with dog_stats_api.timer('lms.grades.CourseGradeFactory.iter', tags=stats_tags):
yield self._iter_grade_result(user, course_data, force_update)
yield self._iter_grade_result(user, course_data, force_update)
def _iter_grade_result(self, user, course_data, force_update):
try:

14
lms/djangoapps/instructor_task/subtasks.py
View File

@@ -12,7 +12,6 @@ from celery.states import READY_STATES, RETRY, SUCCESS
from django.core.cache import cache
from django.db import DatabaseError, transaction
import dogstats_wrapper as dog_stats_api
from util.db import outer_atomic
from .exceptions import DuplicateTaskException
@@ -59,11 +58,6 @@ def track_memory_usage(metric, course_id):
total_memory_info = process.get_memory_info()
total_usage = getattr(total_memory_info, memory_type)
memory_used = total_usage - baseline_usage
dog_stats_api.increment(
metric + "." + memory_type,
memory_used,
tags=["course_id:{}".format(course_id)],
)
def _generate_items_for_subtask(
@@ -410,7 +404,6 @@ def check_subtask_is_valid(entry_id, current_task_id, new_subtask_status):
format_str = "Unexpected task_id '{}': unable to find subtasks of instructor task '{}': rejecting task {}"
msg = format_str.format(current_task_id, entry, new_subtask_status)
TASK_LOG.warning(msg)
dog_stats_api.increment('instructor_task.subtask.duplicate.nosubtasks', tags=[entry.course_id])
raise DuplicateTaskException(msg)
# Confirm that the InstructorTask knows about this particular subtask.
@@ -420,7 +413,6 @@ def check_subtask_is_valid(entry_id, current_task_id, new_subtask_status):
format_str = "Unexpected task_id '{}': unable to find status for subtask of instructor task '{}': rejecting task {}"
msg = format_str.format(current_task_id, entry, new_subtask_status)
TASK_LOG.warning(msg)
dog_stats_api.increment('instructor_task.subtask.duplicate.unknown', tags=[entry.course_id])
raise DuplicateTaskException(msg)
# Confirm that the InstructorTask doesn't think that this subtask has already been
@@ -431,7 +423,6 @@ def check_subtask_is_valid(entry_id, current_task_id, new_subtask_status):
format_str = "Unexpected task_id '{}': already completed - status {} for subtask of instructor task '{}': rejecting task {}"
msg = format_str.format(current_task_id, subtask_status, entry, new_subtask_status)
TASK_LOG.warning(msg)
dog_stats_api.increment('instructor_task.subtask.duplicate.completed', tags=[entry.course_id])
raise DuplicateTaskException(msg)
# Confirm that the InstructorTask doesn't think that this subtask is already being
@@ -445,7 +436,6 @@ def check_subtask_is_valid(entry_id, current_task_id, new_subtask_status):
format_str = "Unexpected task_id '{}': already retried - status {} for subtask of instructor task '{}': rejecting task {}"
msg = format_str.format(current_task_id, subtask_status, entry, new_subtask_status)
TASK_LOG.warning(msg)
dog_stats_api.increment('instructor_task.subtask.duplicate.retried', tags=[entry.course_id])
raise DuplicateTaskException(msg)
# Now we are ready to start working on this. Try to lock it.
@@ -455,7 +445,6 @@ def check_subtask_is_valid(entry_id, current_task_id, new_subtask_status):
format_str = "Unexpected task_id '{}': already being executed - for subtask of instructor task '{}'"
msg = format_str.format(current_task_id, entry)
TASK_LOG.warning(msg)
dog_stats_api.increment('instructor_task.subtask.duplicate.locked', tags=[entry.course_id])
raise DuplicateTaskException(msg)
@@ -479,12 +468,10 @@ def update_subtask_status(entry_id, current_task_id, new_subtask_status, retry_c
if retry_count < MAX_DATABASE_LOCK_RETRIES:
TASK_LOG.info("Retrying to update status for subtask %s of instructor task %d with status %s: retry %d",
current_task_id, entry_id, new_subtask_status, retry_count)
dog_stats_api.increment('instructor_task.subtask.retry_after_failed_update')
update_subtask_status(entry_id, current_task_id, new_subtask_status, retry_count)
else:
TASK_LOG.info("Failed to update status after %d retries for subtask %s of instructor task %d with status %s",
retry_count, current_task_id, entry_id, new_subtask_status)
dog_stats_api.increment('instructor_task.subtask.failed_after_update_retries')
raise
finally:
# Only release the lock on the subtask when we're done trying to update it.
@@ -581,5 +568,4 @@ def _update_subtask_status(entry_id, current_task_id, new_subtask_status):
entry.task_output, current_task_id, entry_id)
except Exception:
TASK_LOG.exception("Unexpected error while updating InstructorTask.")
dog_stats_api.increment('instructor_task.subtask.update_exception')
raise

24
lms/djangoapps/instructor_task/tasks_helper/module_state.py
View File

@@ -8,7 +8,6 @@ from time import time
from django.utils.translation import ugettext_noop
from opaque_keys.edx.keys import UsageKey
import dogstats_wrapper as dog_stats_api
from capa.responsetypes import LoncapaProblemError, ResponseError, StudentInputError
from courseware.courses import get_course_by_id, get_problems_in_section
from courseware.model_data import DjangoKeyValueStore, FieldDataCache
@@ -92,18 +91,17 @@ def perform_module_state_update(update_fcn, filter_fcn, _entry_id, course_id, ta
module_descriptor = problems[unicode(module_to_update.module_state_key)]
# There is no try here: if there's an error, we let it throw, and the task will
# be marked as FAILED, with a stack trace.
with dog_stats_api.timer('instructor_tasks.module.time.step', tags=[u'action:{name}'.format(name=action_name)]):
update_status = update_fcn(module_descriptor, module_to_update, task_input)
if update_status == UPDATE_STATUS_SUCCEEDED:
# If the update_fcn returns true, then it performed some kind of work.
# Logging of failures is left to the update_fcn itself.
task_progress.succeeded += 1
elif update_status == UPDATE_STATUS_FAILED:
task_progress.failed += 1
elif update_status == UPDATE_STATUS_SKIPPED:
task_progress.skipped += 1
else:
raise UpdateProblemModuleStateError("Unexpected update_status returned: {}".format(update_status))
update_status = update_fcn(module_descriptor, module_to_update, task_input)
if update_status == UPDATE_STATUS_SUCCEEDED:
# If the update_fcn returns true, then it performed some kind of work.
# Logging of failures is left to the update_fcn itself.
task_progress.succeeded += 1
elif update_status == UPDATE_STATUS_FAILED:
task_progress.failed += 1
elif update_status == UPDATE_STATUS_SKIPPED:
task_progress.skipped += 1
else:
raise UpdateProblemModuleStateError("Unexpected update_status returned: {}".format(update_status))
return task_progress.update_task_state()

4
lms/djangoapps/instructor_task/tasks_helper/runner.py
View File

@@ -5,7 +5,6 @@ from time import time
from celery import current_task
from django.db import reset_queries
import dogstats_wrapper as dog_stats_api
from lms.djangoapps.instructor_task.models import PROGRESS, InstructorTask
from util.db import outer_atomic
@@ -109,8 +108,7 @@ def run_main_task(entry_id, task_fcn, action_name):
raise ValueError(message)
# Now do the work
with dog_stats_api.timer('instructor_tasks.time.overall', tags=[u'action:{name}'.format(name=action_name)]):
task_progress = task_fcn(entry_id, course_id, task_input, action_name)
task_progress = task_fcn(entry_id, course_id, task_input, action_name)
# Release any queries that the connection has been hanging onto
reset_queries()

1
lms/envs/bok_choy.py
View File

@@ -125,7 +125,6 @@ import logging
LOG_OVERRIDES = [
('track.middleware', logging.CRITICAL),
('edxmako.shortcuts', logging.ERROR),
('dd.dogapi', logging.ERROR),
('edx.discussion', logging.CRITICAL),
]
for log_name, log_level in LOG_OVERRIDES:

3
lms/envs/common.py
View File

@@ -2114,9 +2114,6 @@ INSTALLED_APPS = [
# Splash screen
'splash',
# Monitoring
'openedx.core.djangoapps.datadog.apps.DatadogConfig',
# User API
'rest_framework',
'openedx.core.djangoapps.user_api',

1
lms/envs/devstack.py
View File

@@ -32,7 +32,6 @@ import logging
LOG_OVERRIDES = [
('track.contexts', logging.CRITICAL),
('track.middleware', logging.CRITICAL),
('dd.dogapi', logging.CRITICAL),
('django_comment_client.utils', logging.CRITICAL),
]
for log_name, log_level in LOG_OVERRIDES:

55
lms/lib/comment_client/utils.py
View File

@@ -7,7 +7,6 @@ from uuid import uuid4
import requests
from django.utils.translation import get_language
import dogstats_wrapper as dog_stats_api
from .settings import SERVICE_HOST as COMMENTS_SERVICE
log = logging.getLogger(__name__)
@@ -30,25 +29,6 @@ def extract(dic, keys):
return strip_none({k: dic.get(k) for k in keys})
@contextmanager
def request_timer(request_id, method, url, tags=None):
start = time()
with dog_stats_api.timer('comment_client.request.time', tags=tags):
yield
end = time()
duration = end - start
log.info(
u"comment_client_request_log: request_id={request_id}, method={method}, "
u"url={url}, duration={duration}".format(
request_id=request_id,
method=method,
url=url,
duration=duration
)
)
def perform_request(method, url, data_or_params=None, raw=False,
metric_action=None, metric_tags=None, paged_results=False):
# To avoid dependency conflict
@@ -81,15 +61,14 @@ def perform_request(method, url, data_or_params=None, raw=False,
data = None
params = data_or_params.copy()
params.update(request_id_dict)
with request_timer(request_id, method, url, metric_tags):
response = requests.request(
method,
url,
data=data,
params=params,
headers=headers,
timeout=config.connection_timeout
)
response = requests.request(
method,
url,
data=data,
params=params,
headers=headers,
timeout=config.connection_timeout
)
metric_tags.append(u'status_code:{}'.format(response.status_code))
if response.status_code > 200:
@@ -97,8 +76,6 @@ def perform_request(method, url, data_or_params=None, raw=False,
else:
metric_tags.append(u'result:success')
dog_stats_api.increment('comment_client.request.count', tags=metric_tags)
if 200 < response.status_code < 500:
raise CommentClientRequestError(response.text, response.status_code)
# Heroku returns a 503 when an application is in maintenance mode
@@ -119,22 +96,6 @@ def perform_request(method, url, data_or_params=None, raw=False,
content=response.text[:100]
)
)
if paged_results:
dog_stats_api.histogram(
'comment_client.request.paged.result_count',
value=len(data.get('collection', [])),
tags=metric_tags
)
dog_stats_api.histogram(
'comment_client.request.paged.page',
value=data.get('page', 1),
tags=metric_tags
)
dog_stats_api.histogram(
'comment_client.request.paged.num_pages',
value=data.get('num_pages', 1),
tags=metric_tags
)
return data