From 36269bd5a824cce266cbf3e072d20e16ef1e3209 Mon Sep 17 00:00:00 2001 From: Gabe Mulley Date: Thu, 17 Oct 2013 09:59:51 -0400 Subject: [PATCH] Integrate event-tracking in to the LMS Support incremental conversion of events from the old API to the new, in order to ensure the new system is working, enrollment events have been modified to make use of the new API. --- cms/envs/aws.py | 1 + cms/envs/common.py | 27 +++- common/djangoapps/student/models.py | 4 +- common/djangoapps/student/tests/tests.py | 29 ++--- common/djangoapps/track/middleware.py | 56 ++++++-- common/djangoapps/track/shim.py | 42 ++++++ .../djangoapps/track/tests/test_middleware.py | 101 +++++++++++---- common/djangoapps/track/tests/test_shim.py | 121 ++++++++++++++++++ lms/djangoapps/courseware/features/events.py | 15 +++ .../shoppingcart/tests/test_models.py | 10 +- .../shoppingcart/tests/test_views.py | 9 +- .../verify_student/tests/test_views.py | 21 +-- lms/envs/acceptance.py | 9 ++ lms/envs/aws.py | 1 + lms/envs/common.py | 34 ++++- requirements/edx/github.txt | 2 +- 16 files changed, 387 insertions(+), 95 deletions(-) create mode 100644 common/djangoapps/track/shim.py create mode 100644 common/djangoapps/track/tests/test_shim.py diff --git a/cms/envs/aws.py b/cms/envs/aws.py index 67fc78e644..2a854b7c8a 100644 --- a/cms/envs/aws.py +++ b/cms/envs/aws.py @@ -251,6 +251,7 @@ BROKER_URL = "{0}://{1}:{2}@{3}/{4}".format(CELERY_BROKER_TRANSPORT, # Event tracking TRACKING_BACKENDS.update(AUTH_TOKENS.get("TRACKING_BACKENDS", {})) +EVENT_TRACKING_BACKENDS.update(AUTH_TOKENS.get("EVENT_TRACKING_BACKENDS", {})) SUBDOMAIN_BRANDING = ENV_TOKENS.get('SUBDOMAIN_BRANDING', {}) VIRTUAL_UNIVERSITIES = ENV_TOKENS.get('VIRTUAL_UNIVERSITIES', []) diff --git a/cms/envs/common.py b/cms/envs/common.py index 9e199e3372..85eadd25ca 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -546,7 +546,7 @@ COURSES_WITH_UNSAFE_CODE = [] ############################## EVENT TRACKING ################################# -TRACK_MAX_EVENT = 10000 +TRACK_MAX_EVENT = 50000 TRACKING_BACKENDS = { 'logger': { @@ -557,6 +557,26 @@ TRACKING_BACKENDS = { } } +# We're already logging events, and we don't want to capture user +# names/passwords. Heartbeat events are likely not interesting. +TRACKING_IGNORE_URL_PATTERNS = [r'^/event', r'^/login', r'^/heartbeat'] + +EVENT_TRACKING_ENABLED = True +EVENT_TRACKING_BACKENDS = { + 'logger': { + 'ENGINE': 'eventtracking.backends.logger.LoggerBackend', + 'OPTIONS': { + 'name': 'tracking', + 'max_event_size': TRACK_MAX_EVENT, + } + } +} +EVENT_TRACKING_PROCESSORS = [ + { + 'ENGINE': 'track.shim.LegacyFieldMappingProcessor' + } +] + #### PASSWORD POLICY SETTINGS ##### PASSWORD_MIN_LENGTH = None @@ -565,11 +585,6 @@ PASSWORD_COMPLEXITY = {} PASSWORD_DICTIONARY_EDIT_DISTANCE_THRESHOLD = None PASSWORD_DICTIONARY = [] -# We're already logging events, and we don't want to capture user -# names/passwords. Heartbeat events are likely not interesting. -TRACKING_IGNORE_URL_PATTERNS = [r'^/event', r'^/login', r'^/heartbeat'] -TRACKING_ENABLED = True - ##### ACCOUNT LOCKOUT DEFAULT PARAMETERS ##### MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED = 5 MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS = 15 * 60 diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py index fc6ea6e344..9d8ae8accd 100644 --- a/common/djangoapps/student/models.py +++ b/common/djangoapps/student/models.py @@ -10,7 +10,6 @@ file and check it in at the same time as your model changes. To do that, 2. ./manage.py lms schemamigration student --auto description_of_your_change 3. Add the migration file created in edx-platform/common/djangoapps/student/migrations/ """ -import crum from datetime import datetime, timedelta import hashlib import json @@ -34,7 +33,6 @@ from django.core.exceptions import ObjectDoesNotExist from django.utils.translation import ugettext_noop from django_countries import CountryField from track import contexts -from track.views import server_track from eventtracking import tracker from importlib import import_module @@ -718,7 +716,7 @@ class CourseEnrollment(models.Model): } with tracker.get_tracker().context(event_name, context): - server_track(crum.get_current_request(), event_name, data) + tracker.emit(event_name, data) except: # pylint: disable=bare-except if event_name and self.course_id: log.exception('Unable to emit event %s for user %s and course %s', event_name, self.user.username, self.course_id) diff --git a/common/djangoapps/student/tests/tests.py b/common/djangoapps/student/tests/tests.py index 48d8bb642e..1a8c105c01 100644 --- a/common/djangoapps/student/tests/tests.py +++ b/common/djangoapps/student/tests/tests.py @@ -21,7 +21,7 @@ from xmodule.modulestore.tests.factories import CourseFactory from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from courseware.tests.tests import TEST_DATA_MIXED_MODULESTORE -from mock import Mock, patch, sentinel +from mock import Mock, patch from student.models import anonymous_id_for_user, user_by_anonymous_id, CourseEnrollment, unique_id_for_user from student.views import (process_survey_link, _cert_info, @@ -192,15 +192,10 @@ class EnrollInCourseTest(TestCase): """Tests enrolling and unenrolling in courses.""" def setUp(self): - patcher = patch('student.models.server_track') - self.mock_server_track = patcher.start() + patcher = patch('student.models.tracker') + self.mock_tracker = patcher.start() self.addCleanup(patcher.stop) - crum_patcher = patch('student.models.crum.get_current_request') - self.mock_get_current_request = crum_patcher.start() - self.addCleanup(crum_patcher.stop) - self.mock_get_current_request.return_value = sentinel.request - def test_enrollment(self): user = User.objects.create_user("joe", "joe@joe.com", "password") course_id = "edX/Test101/2013" @@ -254,13 +249,12 @@ class EnrollInCourseTest(TestCase): def assert_no_events_were_emitted(self): """Ensures no events were emitted since the last event related assertion""" - self.assertFalse(self.mock_server_track.called) - self.mock_server_track.reset_mock() + self.assertFalse(self.mock_tracker.emit.called) # pylint: disable=maybe-no-member + self.mock_tracker.reset_mock() def assert_enrollment_event_was_emitted(self, user, course_id): """Ensures an enrollment event was emitted since the last event related assertion""" - self.mock_server_track.assert_called_once_with( - sentinel.request, + self.mock_tracker.emit.assert_called_once_with( # pylint: disable=maybe-no-member 'edx.course.enrollment.activated', { 'course_id': course_id, @@ -268,12 +262,11 @@ class EnrollInCourseTest(TestCase): 'mode': 'honor' } ) - self.mock_server_track.reset_mock() + self.mock_tracker.reset_mock() def assert_unenrollment_event_was_emitted(self, user, course_id): """Ensures an unenrollment event was emitted since the last event related assertion""" - self.mock_server_track.assert_called_once_with( - sentinel.request, + self.mock_tracker.emit.assert_called_once_with( # pylint: disable=maybe-no-member 'edx.course.enrollment.deactivated', { 'course_id': course_id, @@ -281,7 +274,7 @@ class EnrollInCourseTest(TestCase): 'mode': 'honor' } ) - self.mock_server_track.reset_mock() + self.mock_tracker.reset_mock() def test_enrollment_non_existent_user(self): # Testing enrollment of newly unsaved user (i.e. no database entry) @@ -445,8 +438,8 @@ class AnonymousLookupTable(TestCase): mode_slug='honor', mode_display_name='Honor Code', ) - patcher = patch('student.models.server_track') - self.mock_server_track = patcher.start() + patcher = patch('student.models.tracker') + patcher.start() self.addCleanup(patcher.stop) def test_for_unregistered_user(self): # same path as for logged out user diff --git a/common/djangoapps/track/middleware.py b/common/djangoapps/track/middleware.py index 54934c5f36..e32ee9d4ac 100644 --- a/common/djangoapps/track/middleware.py +++ b/common/djangoapps/track/middleware.py @@ -12,6 +12,12 @@ from eventtracking import tracker log = logging.getLogger(__name__) CONTEXT_NAME = 'edx.request' +META_KEY_TO_CONTEXT_KEY = { + 'REMOTE_ADDR': 'ip', + 'SERVER_NAME': 'host', + 'HTTP_USER_AGENT': 'agent', + 'PATH_INFO': 'path' +} class TrackMiddleware(object): @@ -78,26 +84,58 @@ class TrackMiddleware(object): """ Extract information from the request and add it to the tracking context. + + The following fields are injected in to the context: + + * session - The Django session key that identifies the user's session. + * user_id - The numeric ID for the logged in user. + * username - The username of the logged in user. + * ip - The IP address of the client. + * host - The "SERVER_NAME" header, which should be the name of the server running this code. + * agent - The client browser identification string. + * path - The path part of the requested URL. """ - context = {} + context = { + 'session': self.get_session_key(request), + 'user_id': self.get_user_primary_key(request), + 'username': self.get_username(request), + } + for header_name, context_key in META_KEY_TO_CONTEXT_KEY.iteritems(): + context[context_key] = request.META.get(header_name, '') + context.update(contexts.course_context_from_url(request.build_absolute_uri())) - try: - context['user_id'] = request.user.pk - except AttributeError: - context['user_id'] = '' - if settings.DEBUG: - log.error('Cannot determine primary key of logged in user.') tracker.get_tracker().enter_context( CONTEXT_NAME, context ) - def process_response(self, request, response): # pylint: disable=unused-argument + def get_session_key(self, request): + """Gets the Django session key from the request or an empty string if it isn't found""" + try: + return request.session.session_key + except AttributeError: + return '' + + def get_user_primary_key(self, request): + """Gets the primary key of the logged in Django user""" + try: + return request.user.pk + except AttributeError: + return '' + + def get_username(self, request): + """Gets the username of the logged in Django user""" + try: + return request.user.username + except AttributeError: + return '' + + def process_response(self, _request, response): """Exit the context if it exists.""" try: tracker.get_tracker().exit_context(CONTEXT_NAME) - except: # pylint: disable=bare-except + except Exception: # pylint: disable=broad-except pass return response diff --git a/common/djangoapps/track/shim.py b/common/djangoapps/track/shim.py new file mode 100644 index 0000000000..a0849f962b --- /dev/null +++ b/common/djangoapps/track/shim.py @@ -0,0 +1,42 @@ +"""Map new event context values to old top-level field values. Ensures events can be parsed by legacy parsers.""" + +CONTEXT_FIELDS_TO_INCLUDE = [ + 'username', + 'session', + 'ip', + 'agent', + 'host' +] + + +class LegacyFieldMappingProcessor(object): + """Ensures all required fields are included in emitted events""" + + def __call__(self, event): + if 'context' in event: + context = event['context'] + for field in CONTEXT_FIELDS_TO_INCLUDE: + if field in context: + event[field] = context[field] + del context[field] + else: + event[field] = '' + + if 'event_type' in event.get('context', {}): + event['event_type'] = event['context']['event_type'] + del event['context']['event_type'] + else: + event['event_type'] = event.get('name', '') + + if 'data' in event: + event['event'] = event['data'] + del event['data'] + else: + event['event'] = {} + + if 'timestamp' in event: + event['time'] = event['timestamp'] + del event['timestamp'] + + event['event_source'] = 'server' + event['page'] = None diff --git a/common/djangoapps/track/tests/test_middleware.py b/common/djangoapps/track/tests/test_middleware.py index 78d5d46b61..1f4dd3b499 100644 --- a/common/djangoapps/track/tests/test_middleware.py +++ b/common/djangoapps/track/tests/test_middleware.py @@ -1,8 +1,10 @@ import re from mock import patch +from mock import sentinel from django.contrib.auth.models import User +from django.contrib.sessions.middleware import SessionMiddleware from django.test import TestCase from django.test.client import RequestFactory from django.test.utils import override_settings @@ -50,35 +52,86 @@ class TrackMiddlewareTestCase(TestCase): self.track_middleware.process_request(request) self.assertFalse(self.mock_server_track.called) - def test_request_in_course_context(self): - request = self.request_factory.get('/courses/test_org/test_course/test_run/foo') - self.track_middleware.process_request(request) - captured_context = tracker.get_tracker().resolve_context() - self.track_middleware.process_response(request, None) + def test_default_request_context(self): + context = self.get_context_for_path('/courses/') + self.assertEquals(context, { + 'user_id': '', + 'session': '', + 'username': '', + 'ip': '127.0.0.1', + 'host': 'testserver', + 'agent': '', + 'path': '/courses/', + 'org_id': '', + 'course_id': '', + }) + + def get_context_for_path(self, path): + """Extract the generated event tracking context for a given request for the given path.""" + request = self.request_factory.get(path) + return self.get_context_for_request(request) + + def get_context_for_request(self, request): + """Extract the generated event tracking context for the given request.""" + self.track_middleware.process_request(request) + try: + captured_context = tracker.get_tracker().resolve_context() + finally: + self.track_middleware.process_response(request, None) - self.assertEquals( - captured_context, - { - 'course_id': 'test_org/test_course/test_run', - 'org_id': 'test_org', - 'user_id': '' - } - ) self.assertEquals( tracker.get_tracker().resolve_context(), {} ) + return captured_context + + def test_request_in_course_context(self): + captured_context = self.get_context_for_path('/courses/test_org/test_course/test_run/foo') + expected_context_subset = { + 'course_id': 'test_org/test_course/test_run', + 'org_id': 'test_org', + } + self.assert_dict_subset(captured_context, expected_context_subset) + + def assert_dict_subset(self, superset, subset): + """Assert that the superset dict contains all of the key-value pairs found in the subset dict.""" + for key, expected_value in subset.iteritems(): + self.assertEquals(superset[key], expected_value) + def test_request_with_user(self): + user_id = 1 + username = sentinel.username + request = self.request_factory.get('/courses/') - request.user = User(pk=1) - self.track_middleware.process_request(request) - self.addCleanup(self.track_middleware.process_response, request, None) - self.assertEquals( - tracker.get_tracker().resolve_context(), - { - 'course_id': '', - 'org_id': '', - 'user_id': 1 - } - ) + request.user = User(pk=user_id, username=username) + + context = self.get_context_for_request(request) + self.assert_dict_subset(context, { + 'user_id': user_id, + 'username': username, + }) + + def test_request_with_session(self): + request = self.request_factory.get('/courses/') + SessionMiddleware().process_request(request) + request.session.save() + session_key = request.session.session_key + + context = self.get_context_for_request(request) + self.assert_dict_subset(context, { + 'session': session_key, + }) + + def test_request_headers(self): + ip_address = '10.0.0.0' + user_agent = 'UnitTest/1.0' + + factory = RequestFactory(REMOTE_ADDR=ip_address, HTTP_USER_AGENT=user_agent) + request = factory.get('/some-path') + context = self.get_context_for_request(request) + + self.assert_dict_subset(context, { + 'ip': ip_address, + 'agent': user_agent, + }) diff --git a/common/djangoapps/track/tests/test_shim.py b/common/djangoapps/track/tests/test_shim.py new file mode 100644 index 0000000000..b20c513d29 --- /dev/null +++ b/common/djangoapps/track/tests/test_shim.py @@ -0,0 +1,121 @@ +"""Ensure emitted events contain the fields legacy processors expect to find.""" + +from datetime import datetime + +from freezegun import freeze_time +from mock import sentinel +from django.test import TestCase +from django.test.utils import override_settings +from pytz import UTC + +from eventtracking.django import DjangoTracker + + +IN_MEMORY_BACKEND = { + 'mem': { + 'ENGINE': 'track.tests.test_shim.InMemoryBackend' + } +} + +LEGACY_SHIM_PROCESSOR = [ + { + 'ENGINE': 'track.shim.LegacyFieldMappingProcessor' + } +] + +FROZEN_TIME = datetime(2013, 10, 3, 8, 24, 55, tzinfo=UTC) + + +@freeze_time(FROZEN_TIME) +class LegacyFieldMappingProcessorTestCase(TestCase): + """Ensure emitted events contain the fields legacy processors expect to find.""" + + @override_settings( + EVENT_TRACKING_BACKENDS=IN_MEMORY_BACKEND, + EVENT_TRACKING_PROCESSORS=LEGACY_SHIM_PROCESSOR, + ) + def test_event_field_mapping(self): + django_tracker = DjangoTracker() + + data = {sentinel.key: sentinel.value} + + context = { + 'username': sentinel.username, + 'session': sentinel.session, + 'ip': sentinel.ip, + 'host': sentinel.host, + 'agent': sentinel.agent, + 'path': sentinel.path, + 'user_id': sentinel.user_id, + 'course_id': sentinel.course_id, + 'org_id': sentinel.org_id, + 'event_type': sentinel.event_type, + } + with django_tracker.context('test', context): + django_tracker.emit(sentinel.name, data) + + emitted_event = django_tracker.backends['mem'].get_event() + + expected_event = { + 'event_type': sentinel.event_type, + 'name': sentinel.name, + 'context': { + 'user_id': sentinel.user_id, + 'course_id': sentinel.course_id, + 'org_id': sentinel.org_id, + 'path': sentinel.path, + }, + 'event': data, + 'username': sentinel.username, + 'event_source': 'server', + 'time': FROZEN_TIME, + 'agent': sentinel.agent, + 'host': sentinel.host, + 'ip': sentinel.ip, + 'page': None, + 'session': sentinel.session, + } + self.assertEqual(expected_event, emitted_event) + + @override_settings( + EVENT_TRACKING_BACKENDS=IN_MEMORY_BACKEND, + EVENT_TRACKING_PROCESSORS=LEGACY_SHIM_PROCESSOR, + ) + def test_missing_fields(self): + django_tracker = DjangoTracker() + + django_tracker.emit(sentinel.name) + + emitted_event = django_tracker.backends['mem'].get_event() + + expected_event = { + 'event_type': sentinel.name, + 'name': sentinel.name, + 'context': {}, + 'event': {}, + 'username': '', + 'event_source': 'server', + 'time': FROZEN_TIME, + 'agent': '', + 'host': '', + 'ip': '', + 'page': None, + 'session': '', + } + self.assertEqual(expected_event, emitted_event) + + +class InMemoryBackend(object): + """A backend that simply stores all events in memory""" + + def __init__(self): + super(InMemoryBackend, self).__init__() + self.events = [] + + def send(self, event): + """Store the event in a list""" + self.events.append(event) + + def get_event(self): + """Return the first event that was emitted.""" + return self.events[0] diff --git a/lms/djangoapps/courseware/features/events.py b/lms/djangoapps/courseware/features/events.py index 493622f652..62638286a4 100644 --- a/lms/djangoapps/courseware/features/events.py +++ b/lms/djangoapps/courseware/features/events.py @@ -7,6 +7,18 @@ from pymongo import MongoClient from nose.tools import assert_equals from nose.tools import assert_in +REQUIRED_EVENT_FIELDS = [ + 'agent', + 'event', + 'event_source', + 'event_type', + 'host', + 'ip', + 'page', + 'time', + 'username' +] + @before.all def connect_to_mongodb(): @@ -53,3 +65,6 @@ def event_is_emitted(_step, event_type, event_source): } for key, value in expected_field_values.iteritems(): assert_equals(event[key], value) + + for field in REQUIRED_EVENT_FIELDS: + assert_in(field, event) diff --git a/lms/djangoapps/shoppingcart/tests/test_models.py b/lms/djangoapps/shoppingcart/tests/test_models.py index ef210f18df..8f6771987e 100644 --- a/lms/djangoapps/shoppingcart/tests/test_models.py +++ b/lms/djangoapps/shoppingcart/tests/test_models.py @@ -342,13 +342,9 @@ class CertificateItemTest(ModuleStoreTestCase): min_price=self.cost) course_mode.save() - patcher = patch('student.models.server_track') - self.mock_server_track = patcher.start() + patcher = patch('student.models.tracker') + self.mock_tracker = patcher.start() self.addCleanup(patcher.stop) - crum_patcher = patch('student.models.crum.get_current_request') - self.mock_get_current_request = crum_patcher.start() - self.addCleanup(crum_patcher.stop) - self.mock_get_current_request.return_value = sentinel.request def test_existing_enrollment(self): CourseEnrollment.enroll(self.user, self.course_id) @@ -356,7 +352,7 @@ class CertificateItemTest(ModuleStoreTestCase): CertificateItem.add_to_order(cart, self.course_id, self.cost, 'verified') # verify that we are still enrolled self.assertTrue(CourseEnrollment.is_enrolled(self.user, self.course_id)) - self.mock_server_track.reset_mock() + self.mock_tracker.reset_mock() cart.purchase() enrollment = CourseEnrollment.objects.get(user=self.user, course_id=self.course_id) self.assertEquals(enrollment.mode, u'verified') diff --git a/lms/djangoapps/shoppingcart/tests/test_views.py b/lms/djangoapps/shoppingcart/tests/test_views.py index 9799b6ddda..c8e1a67e5d 100644 --- a/lms/djangoapps/shoppingcart/tests/test_views.py +++ b/lms/djangoapps/shoppingcart/tests/test_views.py @@ -40,8 +40,8 @@ postpay_mock = Mock() @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) class ShoppingCartViewsTests(ModuleStoreTestCase): def setUp(self): - patcher = patch('student.models.server_track') - self.mock_server_track = patcher.start() + patcher = patch('student.models.tracker') + self.mock_tracker = patcher.start() self.user = UserFactory.create() self.user.set_password('password') self.user.save() @@ -221,7 +221,7 @@ class ShoppingCartViewsTests(ModuleStoreTestCase): s['attempting_upgrade'] = True s.save() - self.mock_server_track.reset_mock() + self.mock_tracker.emit.reset_mock() # pylint: disable=maybe-no-member resp = self.client.get(reverse('shoppingcart.views.show_receipt', args=[self.cart.id])) # Once they've upgraded, they're no longer *attempting* to upgrade @@ -246,8 +246,7 @@ class ShoppingCartViewsTests(ModuleStoreTestCase): course_enrollment = CourseEnrollment.get_or_create_enrollment(self.user, self.course_id) course_enrollment.emit_event('edx.course.enrollment.upgrade.succeeded') - self.mock_server_track.assert_any_call( - None, + self.mock_tracker.emit.assert_any_call( # pylint: disable=maybe-no-member 'edx.course.enrollment.upgrade.succeeded', { 'user_id': course_enrollment.user.id, diff --git a/lms/djangoapps/verify_student/tests/test_views.py b/lms/djangoapps/verify_student/tests/test_views.py index 28186cfa31..22e406f776 100644 --- a/lms/djangoapps/verify_student/tests/test_views.py +++ b/lms/djangoapps/verify_student/tests/test_views.py @@ -21,8 +21,6 @@ from django.conf import settings from django.core.urlresolvers import reverse from django.core.exceptions import ObjectDoesNotExist -from mock import sentinel - from xmodule.modulestore.tests.factories import CourseFactory from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE from student.tests.factories import UserFactory @@ -133,15 +131,10 @@ class TestMidCourseReverifyView(TestCase): self.course_id = 'Robot/999/Test_Course' CourseFactory.create(org='Robot', number='999', display_name='Test Course') - patcher = patch('student.models.server_track') - self.mock_server_track = patcher.start() + patcher = patch('student.models.tracker') + self.mock_tracker = patcher.start() self.addCleanup(patcher.stop) - crum_patcher = patch('student.models.crum.get_current_request') - self.mock_get_current_request = crum_patcher.start() - self.addCleanup(crum_patcher.stop) - self.mock_get_current_request.return_value = sentinel.request - @patch('verify_student.views.render_to_response', render_mock) def test_midcourse_reverify_get(self): url = reverse('verify_student_midcourse_reverify', @@ -149,8 +142,7 @@ class TestMidCourseReverifyView(TestCase): response = self.client.get(url) # Check that user entering the reverify flow was logged - self.mock_server_track.assert_called_once_with( - sentinel.request, + self.mock_tracker.emit.assert_called_once_with( # pylint: disable=maybe-no-member 'edx.course.enrollment.reverify.started', { 'user_id': self.user.id, @@ -158,7 +150,7 @@ class TestMidCourseReverifyView(TestCase): 'mode': "verified", } ) - self.mock_server_track.reset_mock() + self.mock_tracker.emit.reset_mock() # pylint: disable=maybe-no-member self.assertEquals(response.status_code, 200) ((_template, context), _kwargs) = render_mock.call_args @@ -172,8 +164,7 @@ class TestMidCourseReverifyView(TestCase): response = self.client.post(url, {'face_image': ','}) # Check that submission event was logged - self.mock_server_track.assert_called_once_with( - sentinel.request, + self.mock_tracker.emit.assert_called_once_with( # pylint: disable=maybe-no-member 'edx.course.enrollment.reverify.submitted', { 'user_id': self.user.id, @@ -181,7 +172,7 @@ class TestMidCourseReverifyView(TestCase): 'mode': "verified", } ) - self.mock_server_track.reset_mock() + self.mock_tracker.emit.reset_mock() # pylint: disable=maybe-no-member self.assertEquals(response.status_code, 302) try: diff --git a/lms/envs/acceptance.py b/lms/envs/acceptance.py index ee6256f5fb..d63dad99f4 100644 --- a/lms/envs/acceptance.py +++ b/lms/envs/acceptance.py @@ -88,6 +88,15 @@ TRACKING_BACKENDS.update({ } }) +EVENT_TRACKING_BACKENDS.update({ + 'mongo': { + 'ENGINE': 'eventtracking.backends.mongodb.MongoBackend', + 'OPTIONS': { + 'database': 'track' + } + } +}) + # Enable asset pipeline # Our fork of django-pipeline uses `PIPELINE` instead of `PIPELINE_ENABLED` diff --git a/lms/envs/aws.py b/lms/envs/aws.py index c85b308af0..00705c8894 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -359,6 +359,7 @@ STUDENT_FILEUPLOAD_MAX_SIZE = ENV_TOKENS.get("STUDENT_FILEUPLOAD_MAX_SIZE", STUD # Event tracking TRACKING_BACKENDS.update(AUTH_TOKENS.get("TRACKING_BACKENDS", {})) +EVENT_TRACKING_BACKENDS.update(AUTH_TOKENS.get("EVENT_TRACKING_BACKENDS", {})) # Student identity verification settings VERIFY_STUDENT = AUTH_TOKENS.get("VERIFY_STUDENT", VERIFY_STUDENT) diff --git a/lms/envs/common.py b/lms/envs/common.py index 0368522654..74646b0173 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -394,7 +394,7 @@ LMS_MIGRATION_ALLOWED_IPS = [] ############################## EVENT TRACKING ################################# # FIXME: Should we be doing this truncation? -TRACK_MAX_EVENT = 10000 +TRACK_MAX_EVENT = 50000 DEBUG_TRACK_LOG = False @@ -407,19 +407,39 @@ TRACKING_BACKENDS = { } } +# We're already logging events, and we don't want to capture user +# names/passwords. Heartbeat events are likely not interesting. +TRACKING_IGNORE_URL_PATTERNS = [r'^/event', r'^/login', r'^/heartbeat'] + +EVENT_TRACKING_ENABLED = True +EVENT_TRACKING_BACKENDS = { + 'logger': { + 'ENGINE': 'eventtracking.backends.logger.LoggerBackend', + 'OPTIONS': { + 'name': 'tracking', + 'max_event_size': TRACK_MAX_EVENT, + } + } +} +EVENT_TRACKING_PROCESSORS = [ + { + 'ENGINE': 'track.shim.LegacyFieldMappingProcessor' + } +] + # Backwards compatibility with ENABLE_SQL_TRACKING_LOGS feature flag. -# In the future, adding the backend to TRACKING_BACKENDS enough. +# In the future, adding the backend to TRACKING_BACKENDS should be enough. if FEATURES.get('ENABLE_SQL_TRACKING_LOGS'): TRACKING_BACKENDS.update({ 'sql': { 'ENGINE': 'track.backends.django.DjangoBackend' } }) - -# We're already logging events, and we don't want to capture user -# names/passwords. Heartbeat events are likely not interesting. -TRACKING_IGNORE_URL_PATTERNS = [r'^/event', r'^/login', r'^/heartbeat'] -TRACKING_ENABLED = True + EVENT_TRACKING_BACKENDS.update({ + 'sql': { + 'ENGINE': 'track.backends.django.DjangoBackend' + } + }) ######################## GOOGLE ANALYTICS ########################### GOOGLE_ANALYTICS_ACCOUNT = 'GOOGLE_ANALYTICS_ACCOUNT_DUMMY' diff --git a/requirements/edx/github.txt b/requirements/edx/github.txt index 4128728ca0..3964b022cc 100644 --- a/requirements/edx/github.txt +++ b/requirements/edx/github.txt @@ -22,7 +22,7 @@ -e git+https://github.com/edx/diff-cover.git@v0.2.9#egg=diff_cover -e git+https://github.com/edx/js-test-tool.git@v0.1.5#egg=js_test_tool -e git+https://github.com/edx/django-waffle.git@823a102e48#egg=django-waffle --e git+https://github.com/edx/event-tracking.git@f0211d702d#egg=event-tracking +-e git+https://github.com/edx/event-tracking.git@2ee5ace#egg=event-tracking -e git+https://github.com/edx/bok-choy.git@82b4e82d79b9d4c6d087ebbfa26ea23235728e62#egg=bok_choy -e git+https://github.com/edx-solutions/django-splash.git@9965a53c269666a30bb4e2b3f6037c138aef2a55#egg=django-splash -e git+https://github.com/edx/acid-block.git@459aff7b63db8f2c5decd1755706c1a64fb4ebb1#egg=acid-xblock