Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove dogstatsd-python in favor of dogapi

Browse Source 
Also added datadog application to commom django apps and set it to use
a local statsd server by default.
This commit is contained in:
Carlos Andrés Rocha
2013-09-09 16:15:24 -04:00
parent 0734d835a1
commit ee0004e2f8
15 changed files with 97 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
cms/envs/aws.py
View File

@@ -147,7 +147,12 @@ MODULESTORE = AUTH_TOKENS['MODULESTORE']
CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE']
# Datadog for events!
DATADOG_API = AUTH_TOKENS.get("DATADOG_API")
DATADOG = AUTH_TOKENS.get("DATADOG", {})
DATADOG = DATADOG.update(ENV_TOKENS.get("DATADOG", {}))
# TODO: deprecated (compatibility with previous settings)
if 'DATADOG_API' in AUTH_TOKENS:
DATADOG['api_key'] = AUTH_TOKENS['DATADOG_API']
# Celery Broker
CELERY_BROKER_TRANSPORT = ENV_TOKENS.get("CELERY_BROKER_TRANSPORT", "")

3
cms/envs/common.py
View File

@@ -358,6 +358,9 @@ INSTALLED_APPS = (
# Tracking
'track',
# Monitoring
'datadog',
# For asset pipelining
'mitxmako',
'pipeline',

0
common/djangoapps/datadog/__init__.py Normal file
View File

23
common/djangoapps/datadog/startup.py
View File

@@ -1,12 +1,25 @@
from django.conf import settings
from dogapi import dog_http_api, dog_stats_api
from dogapi import dog_stats_api, dog_http_api
def run():
"""
Initialize connection to datadog during django startup.
Expects the datadog api key in the DATADOG_API settings key
Can be configured using a dictionary named DATADOG in the django
project settings.
"""
if hasattr(settings, 'DATADOG_API'):
dog_http_api.api_key = settings.DATADOG_API
dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True)
# By default use the statsd agent
options = {'statsd': True}
if hasattr(settings, 'DATADOG'):
options.update(settings.DATADOG)
# Not all arguments are documented.
# Look at the source code for details.
dog_stats_api.start(**options)
dog_http_api.api_key = options.get('api_key')

13
common/djangoapps/student/management/commands/pearson_import_conf_zip.py
View File

@@ -1,21 +1,24 @@
import csv
from zipfile import ZipFile, is_zipfile
from time import strptime, strftime
from datetime import datetime
from zipfile import ZipFile, is_zipfile
from dogapi import dog_http_api
from pytz import UTC
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
import django_startup
from student.models import TestCenterUser, TestCenterRegistration
from pytz import UTC
django_startup.autostartup()
class Command(BaseCommand):
dog_http_api.api_key = settings.DATADOG_API
args = '<input zip file>'
help = """
Import Pearson confirmation files and update TestCenterUser

17
common/djangoapps/student/management/commands/pearson_transfer.py
View File

@@ -1,16 +1,19 @@
import os
from optparse import make_option
import os
from stat import S_ISDIR
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.core.management import call_command
import boto
from dogapi import dog_http_api, dog_stats_api
import paramiko
import boto
dog_http_api.api_key = settings.DATADOG_API
dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True)
from django.conf import settings
from django.core.management import call_command
from django.core.management.base import BaseCommand, CommandError
import django_startup
django_startup.autostartup()
class Command(BaseCommand):

28
common/djangoapps/student/management/commands/tests/test_pearson.py
View File

@@ -303,15 +303,13 @@ class PearsonTransferTestCase(PearsonTestCase):
'''
def test_transfer_config(self):
with self.settings(DATADOG_API='FAKE_KEY'):
# TODO: why is this failing with the wrong error message?!
stderrmsg = get_command_error_text('pearson_transfer', **{'mode': 'garbage'})
self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
with self.settings(DATADOG_API='FAKE_KEY'):
stderrmsg = get_command_error_text('pearson_transfer')
self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
with self.settings(DATADOG_API='FAKE_KEY',
PEARSON={'LOCAL_EXPORT': self.export_dir,
stderrmsg = get_command_error_text('pearson_transfer', **{'mode': 'garbage'})
self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
stderrmsg = get_command_error_text('pearson_transfer')
self.assertErrorContains(stderrmsg, 'Error: No PEARSON entries')
with self.settings(PEARSON={'LOCAL_EXPORT': self.export_dir,
'LOCAL_IMPORT': self.import_dir}):
stderrmsg = get_command_error_text('pearson_transfer')
self.assertErrorContains(stderrmsg, 'Error: No entry in the PEARSON settings')
@@ -319,8 +317,7 @@ class PearsonTransferTestCase(PearsonTestCase):
def test_transfer_export_missing_dest_dir(self):
raise SkipTest()
create_multiple_registrations('export_missing_dest')
with self.settings(DATADOG_API='FAKE_KEY',
PEARSON={'LOCAL_EXPORT': self.export_dir,
with self.settings(PEARSON={'LOCAL_EXPORT': self.export_dir,
'SFTP_EXPORT': 'this/does/not/exist',
'SFTP_HOSTNAME': SFTP_HOSTNAME,
'SFTP_USERNAME': SFTP_USERNAME,
@@ -336,8 +333,7 @@ class PearsonTransferTestCase(PearsonTestCase):
def test_transfer_export(self):
raise SkipTest()
create_multiple_registrations("transfer_export")
with self.settings(DATADOG_API='FAKE_KEY',
PEARSON={'LOCAL_EXPORT': self.export_dir,
with self.settings(PEARSON={'LOCAL_EXPORT': self.export_dir,
'SFTP_EXPORT': 'results/topvue',
'SFTP_HOSTNAME': SFTP_HOSTNAME,
'SFTP_USERNAME': SFTP_USERNAME,
@@ -354,8 +350,7 @@ class PearsonTransferTestCase(PearsonTestCase):
def test_transfer_import_missing_source_dir(self):
raise SkipTest()
create_multiple_registrations('import_missing_src')
with self.settings(DATADOG_API='FAKE_KEY',
PEARSON={'LOCAL_IMPORT': self.import_dir,
with self.settings(PEARSON={'LOCAL_IMPORT': self.import_dir,
'SFTP_IMPORT': 'this/does/not/exist',
'SFTP_HOSTNAME': SFTP_HOSTNAME,
'SFTP_USERNAME': SFTP_USERNAME,
@@ -371,8 +366,7 @@ class PearsonTransferTestCase(PearsonTestCase):
def test_transfer_import(self):
raise SkipTest()
create_multiple_registrations('import_missing_src')
with self.settings(DATADOG_API='FAKE_KEY',
PEARSON={'LOCAL_IMPORT': self.import_dir,
with self.settings(PEARSON={'LOCAL_IMPORT': self.import_dir,
'SFTP_IMPORT': 'results',
'SFTP_HOSTNAME': SFTP_HOSTNAME,
'SFTP_USERNAME': SFTP_USERNAME,

26
common/djangoapps/student/views.py
View File

@@ -59,7 +59,7 @@ from bulk_email.models import Optout
import track.views
from statsd import statsd
from dogapi import dog_stats_api
from pytz import UTC
log = logging.getLogger("mitx.student")
@@ -388,10 +388,12 @@ def change_enrollment(request):
)
org, course_num, run = course_id.split("/")
statsd.increment("common.student.enrollment",
tags=["org:{0}".format(org),
"course:{0}".format(course_num),
"run:{0}".format(run)])
dog_stats_api.increment(
"common.student.enrollment",
tags=["org:{0}".format(org),
"course:{0}".format(course_num),
"run:{0}".format(run)]
)
CourseEnrollment.enroll(user, course.id)
@@ -402,10 +404,12 @@ def change_enrollment(request):
CourseEnrollment.unenroll(user, course_id)
org, course_num, run = course_id.split("/")
statsd.increment("common.student.unenrollment",
tags=["org:{0}".format(org),
"course:{0}".format(course_num),
"run:{0}".format(run)])
dog_stats_api.increment(
"common.student.unenrollment",
tags=["org:{0}".format(org),
"course:{0}".format(course_num),
"run:{0}".format(run)]
)
return HttpResponse()
except CourseEnrollment.DoesNotExist:
@@ -471,7 +475,7 @@ def login_user(request, error=""):
redirect_url = try_change_enrollment(request)
statsd.increment("common.student.successful_login")
dog_stats_api.increment("common.student.successful_login")
response = HttpResponse(json.dumps({'success': True, 'redirect_url': redirect_url}))
# set the login cookie for the edx marketing site
@@ -740,7 +744,7 @@ def create_account(request, post_override=None):
redirect_url = try_change_enrollment(request)
statsd.increment("common.student.account_created")
dog_stats_api.increment("common.student.account_created")
response_params = {'success': True,
'redirect_url': redirect_url}

4
common/lib/capa/capa/safe_exec/safe_exec.py
View File

@@ -4,7 +4,7 @@ from codejail.safe_exec import safe_exec as codejail_safe_exec
from codejail.safe_exec import not_safe_exec as codejail_not_safe_exec
from codejail.safe_exec import json_safe, SafeExecException
from . import lazymod
from statsd import statsd
from dogapi import dog_stats_api
import hashlib
@@ -70,7 +70,7 @@ def update_hash(hasher, obj):
hasher.update(repr(obj))
@statsd.timed('capa.safe_exec.time')
@dog_stats_api.timed('capa.safe_exec.time')
def safe_exec(code, globals_dict, random_seed=None, python_path=None, cache=None, slug=None, unsafely=False):
"""
Execute python code safely.

7
lms/djangoapps/bulk_email/tasks.py
View File

@@ -6,6 +6,7 @@ import math
import re
import time
from dogapi import dog_stats_api
from smtplib import SMTPServerDisconnected, SMTPDataError, SMTPConnectError
from django.conf import settings
@@ -15,8 +16,6 @@ from django.http import Http404
from celery import task, current_task
from celery.utils.log import get_task_logger
from django.core.urlresolvers import reverse
from statsd import statsd
from dogapi import dog_stats_api
from bulk_email.models import (
CourseEmail, Optout, CourseEmailTemplate,
@@ -192,7 +191,7 @@ def _send_course_email(email_id, to_list, course_title, course_url, image_url, t
with dog_stats_api.timer('course_email.single_send.time.overall', tags=[_statsd_tag(course_title)]):
connection.send_messages([email_msg])
statsd.increment('course_email.sent', tags=[_statsd_tag(course_title)])
dog_stats_api.increment('course_email.sent', tags=[_statsd_tag(course_title)])
log.info('Email with id %s sent to %s', email_id, email)
num_sent += 1
@@ -205,7 +204,7 @@ def _send_course_email(email_id, to_list, course_title, course_url, image_url, t
# This will fall through and not retry the message, since it will be popped
log.warning('Email with id %s not delivered to %s due to error %s', email_id, email, exc.smtp_error)
statsd.increment('course_email.error', tags=[_statsd_tag(course_title)])
dog_stats_api.increment('course_email.error', tags=[_statsd_tag(course_title)])
num_error += 1

4
lms/djangoapps/courseware/module_render.py
View File

@@ -13,7 +13,7 @@ from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt
from requests.auth import HTTPBasicAuth
from statsd import statsd
from dogapi import dog_stats_api
from capa.xqueue_interface import XQueueInterface
from mitxmako.shortcuts import render_to_string
@@ -332,7 +332,7 @@ def get_module_for_descriptor_internal(user, descriptor, field_data_cache, cours
if grade_bucket_type is not None:
tags.append('type:%s' % grade_bucket_type)
statsd.increment("lms.courseware.question_answered", tags=tags)
dog_stats_api.increment("lms.courseware.question_answered", tags=tags)
# TODO (cpennington): When modules are shared between courses, the static
# prefix is going to have to be specific to the module, not the directory

12
lms/djangoapps/shoppingcart/models.py
View File

@@ -18,7 +18,7 @@ from courseware.courses import get_course_about_section
from mitxmako.shortcuts import render_to_string
from student.views import course_from_id
from student.models import CourseEnrollment
from statsd import statsd
from dogapi import dog_stats_api
from xmodule.modulestore.django import modulestore
from xmodule.course_module import CourseDescriptor
@@ -300,10 +300,12 @@ class PaidCourseRegistration(OrderItem):
log.info("Enrolled {0} in paid course {1}, paid ${2}".format(self.user.email, self.course_id, self.line_cost))
org, course_num, run = self.course_id.split("/")
statsd.increment("shoppingcart.PaidCourseRegistration.purchased_callback.enrollment",
tags=["org:{0}".format(org),
"course:{0}".format(course_num),
"run:{0}".format(run)])
dog_stats_api.increment(
"shoppingcart.PaidCourseRegistration.purchased_callback.enrollment",
tags=["org:{0}".format(org),
"course:{0}".format(course_num),
"run:{0}".format(run)]
)
class CertificateItem(OrderItem):

7
lms/envs/aws.py
View File

@@ -229,7 +229,12 @@ PEARSON_TEST_PASSWORD = AUTH_TOKENS.get("PEARSON_TEST_PASSWORD")
PEARSON = AUTH_TOKENS.get("PEARSON")
# Datadog for events!
DATADOG_API = AUTH_TOKENS.get("DATADOG_API")
DATADOG = AUTH_TOKENS.get("DATADOG", {})
DATADOG = DATADOG.update(ENV_TOKENS.get("DATADOG", {}))
# TODO: deprecated (compatibility with previous settings)
if 'DATADOG_API' in AUTH_TOKENS:
DATADOG['api_key'] = AUTH_TOKENS['DATADOG_API']
# Analytics dashboard server
ANALYTICS_SERVER_URL = ENV_TOKENS.get("ANALYTICS_SERVER_URL")

9
lms/envs/common.py
View File

@@ -790,7 +790,7 @@ INSTALLED_APPS = (
'external_auth',
'django_openid_auth',
#For the wiki
# For the wiki
'wiki', # The new django-wiki from benjaoming
'django_notify',
'course_wiki', # Our customizations
@@ -801,7 +801,7 @@ INSTALLED_APPS = (
'wiki.plugins.notifications',
'course_wiki.plugins.markdownedx',
# foldit integration
# Foldit integration
'foldit',
# For testing
@@ -814,11 +814,14 @@ INSTALLED_APPS = (
'django_comment_common',
'notes',
# Monitoring
'datadog',
# User API
'rest_framework',
'user_api',
# shopping cart
# Shopping cart
'shoppingcart',
# Notification preferences setting

1
requirements/edx/base.txt
View File

@@ -70,7 +70,6 @@ watchdog==0.6.0
# Metrics gathering and monitoring
dogapi==1.2.1
dogstatsd-python==0.2.1
newrelic==1.13.1.31
# Used for documentation gathering