From ee0004e2f86b684418c8d75adc9ab0839fafd20c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Andr=C3=A9s=20Rocha?= Date: Mon, 9 Sep 2013 16:15:24 -0400 Subject: [PATCH] Remove dogstatsd-python in favor of dogapi Also added datadog application to commom django apps and set it to use a local statsd server by default. --- cms/envs/aws.py | 7 ++++- cms/envs/common.py | 3 ++ common/djangoapps/datadog/__init__.py | 0 common/djangoapps/datadog/startup.py | 23 +++++++++++---- .../commands/pearson_import_conf_zip.py | 13 +++++---- .../management/commands/pearson_transfer.py | 17 ++++++----- .../management/commands/tests/test_pearson.py | 28 ++++++++----------- common/djangoapps/student/views.py | 26 +++++++++-------- common/lib/capa/capa/safe_exec/safe_exec.py | 4 +-- lms/djangoapps/bulk_email/tasks.py | 7 ++--- lms/djangoapps/courseware/module_render.py | 4 +-- lms/djangoapps/shoppingcart/models.py | 12 ++++---- lms/envs/aws.py | 7 ++++- lms/envs/common.py | 9 ++++-- requirements/edx/base.txt | 1 - 15 files changed, 97 insertions(+), 64 deletions(-) create mode 100644 common/djangoapps/datadog/__init__.py diff --git a/cms/envs/aws.py b/cms/envs/aws.py index c2ba51a5f8..eb9a2e0c72 100644 --- a/cms/envs/aws.py +++ b/cms/envs/aws.py @@ -147,7 +147,12 @@ MODULESTORE = AUTH_TOKENS['MODULESTORE'] CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE'] # Datadog for events! -DATADOG_API = AUTH_TOKENS.get("DATADOG_API") +DATADOG = AUTH_TOKENS.get("DATADOG", {}) +DATADOG = DATADOG.update(ENV_TOKENS.get("DATADOG", {})) + +# TODO: deprecated (compatibility with previous settings) +if 'DATADOG_API' in AUTH_TOKENS: + DATADOG['api_key'] = AUTH_TOKENS['DATADOG_API'] # Celery Broker CELERY_BROKER_TRANSPORT = ENV_TOKENS.get("CELERY_BROKER_TRANSPORT", "") diff --git a/cms/envs/common.py b/cms/envs/common.py index 9b9f58a7b5..39995b267c 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -358,6 +358,9 @@ INSTALLED_APPS = ( # Tracking 'track', + # Monitoring + 'datadog', + # For asset pipelining 'mitxmako', 'pipeline', diff --git a/common/djangoapps/datadog/__init__.py b/common/djangoapps/datadog/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/common/djangoapps/datadog/startup.py b/common/djangoapps/datadog/startup.py index 41949c3a94..2767eb5dfc 100644 --- a/common/djangoapps/datadog/startup.py +++ b/common/djangoapps/datadog/startup.py @@ -1,12 +1,25 @@ from django.conf import settings -from dogapi import dog_http_api, dog_stats_api + +from dogapi import dog_stats_api, dog_http_api + def run(): """ Initialize connection to datadog during django startup. - Expects the datadog api key in the DATADOG_API settings key + Can be configured using a dictionary named DATADOG in the django + project settings. + """ - if hasattr(settings, 'DATADOG_API'): - dog_http_api.api_key = settings.DATADOG_API - dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True) + + # By default use the statsd agent + options = {'statsd': True} + + if hasattr(settings, 'DATADOG'): + options.update(settings.DATADOG) + + # Not all arguments are documented. + # Look at the source code for details. + dog_stats_api.start(**options) + + dog_http_api.api_key = options.get('api_key') diff --git a/common/djangoapps/student/management/commands/pearson_import_conf_zip.py b/common/djangoapps/student/management/commands/pearson_import_conf_zip.py index 1e06a0931a..3edb3a76d7 100644 --- a/common/djangoapps/student/management/commands/pearson_import_conf_zip.py +++ b/common/djangoapps/student/management/commands/pearson_import_conf_zip.py @@ -1,21 +1,24 @@ import csv - -from zipfile import ZipFile, is_zipfile from time import strptime, strftime - from datetime import datetime +from zipfile import ZipFile, is_zipfile + from dogapi import dog_http_api +from pytz import UTC from django.core.management.base import BaseCommand, CommandError from django.conf import settings +import django_startup + from student.models import TestCenterUser, TestCenterRegistration -from pytz import UTC + + +django_startup.autostartup() class Command(BaseCommand): - dog_http_api.api_key = settings.DATADOG_API args = '' help = """ Import Pearson confirmation files and update TestCenterUser diff --git a/common/djangoapps/student/management/commands/pearson_transfer.py b/common/djangoapps/student/management/commands/pearson_transfer.py index 39e1e7468e..b00cf27ffb 100644 --- a/common/djangoapps/student/management/commands/pearson_transfer.py +++ b/common/djangoapps/student/management/commands/pearson_transfer.py @@ -1,16 +1,19 @@ -import os from optparse import make_option +import os from stat import S_ISDIR -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError -from django.core.management import call_command +import boto from dogapi import dog_http_api, dog_stats_api import paramiko -import boto -dog_http_api.api_key = settings.DATADOG_API -dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True) +from django.conf import settings +from django.core.management import call_command +from django.core.management.base import BaseCommand, CommandError + +import django_startup + + +django_startup.autostartup() class Command(BaseCommand): diff --git a/common/djangoapps/student/management/commands/tests/test_pearson.py b/common/djangoapps/student/management/commands/tests/test_pearson.py index ca6e20673b..68fa10eaaa 100644 --- a/common/djangoapps/student/management/commands/tests/test_pearson.py +++ b/common/djangoapps/student/management/commands/tests/test_pearson.py @@ -303,15 +303,13 @@ class PearsonTransferTestCase(PearsonTestCase): ''' def test_transfer_config(self): - with self.settings(DATADOG_API='FAKE_KEY'): - # TODO: why is this failing with the wrong error message?! - stderrmsg = get_command_error_text('pearson_transfer', **{'mode': 'garbage'}) - self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries') - with self.settings(DATADOG_API='FAKE_KEY'): - stderrmsg = get_command_error_text('pearson_transfer') - self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries') - with self.settings(DATADOG_API='FAKE_KEY', - PEARSON={'LOCAL_EXPORT': self.export_dir, + stderrmsg = get_command_error_text('pearson_transfer', **{'mode': 'garbage'}) + self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries') + + stderrmsg = get_command_error_text('pearson_transfer') + self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries') + + with self.settings(PEARSON={'LOCAL_EXPORT': self.export_dir, 'LOCAL_IMPORT': self.import_dir}): stderrmsg = get_command_error_text('pearson_transfer') self.assertErrorContains(stderrmsg, 'Error: No entry in the PEARSON settings') @@ -319,8 +317,7 @@ class PearsonTransferTestCase(PearsonTestCase): def test_transfer_export_missing_dest_dir(self): raise SkipTest() create_multiple_registrations('export_missing_dest') - with self.settings(DATADOG_API='FAKE_KEY', - PEARSON={'LOCAL_EXPORT': self.export_dir, + with self.settings(PEARSON={'LOCAL_EXPORT': self.export_dir, 'SFTP_EXPORT': 'this/does/not/exist', 'SFTP_HOSTNAME': SFTP_HOSTNAME, 'SFTP_USERNAME': SFTP_USERNAME, @@ -336,8 +333,7 @@ class PearsonTransferTestCase(PearsonTestCase): def test_transfer_export(self): raise SkipTest() create_multiple_registrations("transfer_export") - with self.settings(DATADOG_API='FAKE_KEY', - PEARSON={'LOCAL_EXPORT': self.export_dir, + with self.settings(PEARSON={'LOCAL_EXPORT': self.export_dir, 'SFTP_EXPORT': 'results/topvue', 'SFTP_HOSTNAME': SFTP_HOSTNAME, 'SFTP_USERNAME': SFTP_USERNAME, @@ -354,8 +350,7 @@ class PearsonTransferTestCase(PearsonTestCase): def test_transfer_import_missing_source_dir(self): raise SkipTest() create_multiple_registrations('import_missing_src') - with self.settings(DATADOG_API='FAKE_KEY', - PEARSON={'LOCAL_IMPORT': self.import_dir, + with self.settings(PEARSON={'LOCAL_IMPORT': self.import_dir, 'SFTP_IMPORT': 'this/does/not/exist', 'SFTP_HOSTNAME': SFTP_HOSTNAME, 'SFTP_USERNAME': SFTP_USERNAME, @@ -371,8 +366,7 @@ class PearsonTransferTestCase(PearsonTestCase): def test_transfer_import(self): raise SkipTest() create_multiple_registrations('import_missing_src') - with self.settings(DATADOG_API='FAKE_KEY', - PEARSON={'LOCAL_IMPORT': self.import_dir, + with self.settings(PEARSON={'LOCAL_IMPORT': self.import_dir, 'SFTP_IMPORT': 'results', 'SFTP_HOSTNAME': SFTP_HOSTNAME, 'SFTP_USERNAME': SFTP_USERNAME, diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py index 78691f9134..8dad1d828a 100644 --- a/common/djangoapps/student/views.py +++ b/common/djangoapps/student/views.py @@ -59,7 +59,7 @@ from bulk_email.models import Optout import track.views -from statsd import statsd +from dogapi import dog_stats_api from pytz import UTC log = logging.getLogger("mitx.student") @@ -388,10 +388,12 @@ def change_enrollment(request): ) org, course_num, run = course_id.split("/") - statsd.increment("common.student.enrollment", - tags=["org:{0}".format(org), - "course:{0}".format(course_num), - "run:{0}".format(run)]) + dog_stats_api.increment( + "common.student.enrollment", + tags=["org:{0}".format(org), + "course:{0}".format(course_num), + "run:{0}".format(run)] + ) CourseEnrollment.enroll(user, course.id) @@ -402,10 +404,12 @@ def change_enrollment(request): CourseEnrollment.unenroll(user, course_id) org, course_num, run = course_id.split("/") - statsd.increment("common.student.unenrollment", - tags=["org:{0}".format(org), - "course:{0}".format(course_num), - "run:{0}".format(run)]) + dog_stats_api.increment( + "common.student.unenrollment", + tags=["org:{0}".format(org), + "course:{0}".format(course_num), + "run:{0}".format(run)] + ) return HttpResponse() except CourseEnrollment.DoesNotExist: @@ -471,7 +475,7 @@ def login_user(request, error=""): redirect_url = try_change_enrollment(request) - statsd.increment("common.student.successful_login") + dog_stats_api.increment("common.student.successful_login") response = HttpResponse(json.dumps({'success': True, 'redirect_url': redirect_url})) # set the login cookie for the edx marketing site @@ -740,7 +744,7 @@ def create_account(request, post_override=None): redirect_url = try_change_enrollment(request) - statsd.increment("common.student.account_created") + dog_stats_api.increment("common.student.account_created") response_params = {'success': True, 'redirect_url': redirect_url} diff --git a/common/lib/capa/capa/safe_exec/safe_exec.py b/common/lib/capa/capa/safe_exec/safe_exec.py index be33bcaa5b..01551ffcb8 100644 --- a/common/lib/capa/capa/safe_exec/safe_exec.py +++ b/common/lib/capa/capa/safe_exec/safe_exec.py @@ -4,7 +4,7 @@ from codejail.safe_exec import safe_exec as codejail_safe_exec from codejail.safe_exec import not_safe_exec as codejail_not_safe_exec from codejail.safe_exec import json_safe, SafeExecException from . import lazymod -from statsd import statsd +from dogapi import dog_stats_api import hashlib @@ -70,7 +70,7 @@ def update_hash(hasher, obj): hasher.update(repr(obj)) -@statsd.timed('capa.safe_exec.time') +@dog_stats_api.timed('capa.safe_exec.time') def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None, slug=None, unsafely=False): """ Execute python code safely. diff --git a/lms/djangoapps/bulk_email/tasks.py b/lms/djangoapps/bulk_email/tasks.py index f3e8a9f4fa..1f15a8b4d7 100644 --- a/lms/djangoapps/bulk_email/tasks.py +++ b/lms/djangoapps/bulk_email/tasks.py @@ -6,6 +6,7 @@ import math import re import time +from dogapi import dog_stats_api from smtplib import SMTPServerDisconnected, SMTPDataError, SMTPConnectError from django.conf import settings @@ -15,8 +16,6 @@ from django.http import Http404 from celery import task, current_task from celery.utils.log import get_task_logger from django.core.urlresolvers import reverse -from statsd import statsd -from dogapi import dog_stats_api from bulk_email.models import ( CourseEmail, Optout, CourseEmailTemplate, @@ -192,7 +191,7 @@ def _send_course_email(email_id, to_list, course_title, course_url, image_url, t with dog_stats_api.timer('course_email.single_send.time.overall', tags=[_statsd_tag(course_title)]): connection.send_messages([email_msg]) - statsd.increment('course_email.sent', tags=[_statsd_tag(course_title)]) + dog_stats_api.increment('course_email.sent', tags=[_statsd_tag(course_title)]) log.info('Email with id %s sent to %s', email_id, email) num_sent += 1 @@ -205,7 +204,7 @@ def _send_course_email(email_id, to_list, course_title, course_url, image_url, t # This will fall through and not retry the message, since it will be popped log.warning('Email with id %s not delivered to %s due to error %s', email_id, email, exc.smtp_error) - statsd.increment('course_email.error', tags=[_statsd_tag(course_title)]) + dog_stats_api.increment('course_email.error', tags=[_statsd_tag(course_title)]) num_error += 1 diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 53f9c57f38..1df849bc22 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -13,7 +13,7 @@ from django.http import HttpResponse from django.views.decorators.csrf import csrf_exempt from requests.auth import HTTPBasicAuth -from statsd import statsd +from dogapi import dog_stats_api from capa.xqueue_interface import XQueueInterface from mitxmako.shortcuts import render_to_string @@ -332,7 +332,7 @@ def get_module_for_descriptor_internal(user, descriptor, field_data_cache, cours if grade_bucket_type is not None: tags.append('type:%s' % grade_bucket_type) - statsd.increment("lms.courseware.question_answered", tags=tags) + dog_stats_api.increment("lms.courseware.question_answered", tags=tags) # TODO (cpennington): When modules are shared between courses, the static # prefix is going to have to be specific to the module, not the directory diff --git a/lms/djangoapps/shoppingcart/models.py b/lms/djangoapps/shoppingcart/models.py index d613903057..bb8c00fc51 100644 --- a/lms/djangoapps/shoppingcart/models.py +++ b/lms/djangoapps/shoppingcart/models.py @@ -18,7 +18,7 @@ from courseware.courses import get_course_about_section from mitxmako.shortcuts import render_to_string from student.views import course_from_id from student.models import CourseEnrollment -from statsd import statsd +from dogapi import dog_stats_api from xmodule.modulestore.django import modulestore from xmodule.course_module import CourseDescriptor @@ -300,10 +300,12 @@ class PaidCourseRegistration(OrderItem): log.info("Enrolled {0} in paid course {1}, paid ${2}".format(self.user.email, self.course_id, self.line_cost)) org, course_num, run = self.course_id.split("/") - statsd.increment("shoppingcart.PaidCourseRegistration.purchased_callback.enrollment", - tags=["org:{0}".format(org), - "course:{0}".format(course_num), - "run:{0}".format(run)]) + dog_stats_api.increment( + "shoppingcart.PaidCourseRegistration.purchased_callback.enrollment", + tags=["org:{0}".format(org), + "course:{0}".format(course_num), + "run:{0}".format(run)] + ) class CertificateItem(OrderItem): diff --git a/lms/envs/aws.py b/lms/envs/aws.py index 35275036e5..d608933f19 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -229,7 +229,12 @@ PEARSON_TEST_PASSWORD = AUTH_TOKENS.get("PEARSON_TEST_PASSWORD") PEARSON = AUTH_TOKENS.get("PEARSON") # Datadog for events! -DATADOG_API = AUTH_TOKENS.get("DATADOG_API") +DATADOG = AUTH_TOKENS.get("DATADOG", {}) +DATADOG = DATADOG.update(ENV_TOKENS.get("DATADOG", {})) + +# TODO: deprecated (compatibility with previous settings) +if 'DATADOG_API' in AUTH_TOKENS: + DATADOG['api_key'] = AUTH_TOKENS['DATADOG_API'] # Analytics dashboard server ANALYTICS_SERVER_URL = ENV_TOKENS.get("ANALYTICS_SERVER_URL") diff --git a/lms/envs/common.py b/lms/envs/common.py index 230b90a203..580c3b4e34 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -790,7 +790,7 @@ INSTALLED_APPS = ( 'external_auth', 'django_openid_auth', - #For the wiki + # For the wiki 'wiki', # The new django-wiki from benjaoming 'django_notify', 'course_wiki', # Our customizations @@ -801,7 +801,7 @@ INSTALLED_APPS = ( 'wiki.plugins.notifications', 'course_wiki.plugins.markdownedx', - # foldit integration + # Foldit integration 'foldit', # For testing @@ -814,11 +814,14 @@ INSTALLED_APPS = ( 'django_comment_common', 'notes', + # Monitoring + 'datadog', + # User API 'rest_framework', 'user_api', - # shopping cart + # Shopping cart 'shoppingcart', # Notification preferences setting diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 77fbe40ecc..d3bbdd1502 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -70,7 +70,6 @@ watchdog==0.6.0 # Metrics gathering and monitoring dogapi==1.2.1 -dogstatsd-python==0.2.1 newrelic==1.13.1.31 # Used for documentation gathering