diff --git a/cms/envs/aws.py b/cms/envs/aws.py index 319ed99e46..ea9b758003 100644 --- a/cms/envs/aws.py +++ b/cms/envs/aws.py @@ -280,7 +280,9 @@ BROKER_URL = "{0}://{1}:{2}@{3}/{4}".format(CELERY_BROKER_TRANSPORT, # Event tracking TRACKING_BACKENDS.update(AUTH_TOKENS.get("TRACKING_BACKENDS", {})) -EVENT_TRACKING_BACKENDS.update(AUTH_TOKENS.get("EVENT_TRACKING_BACKENDS", {})) +EVENT_TRACKING_BACKENDS['tracking_logs']['OPTIONS']['backends'].update(AUTH_TOKENS.get("EVENT_TRACKING_BACKENDS", {})) +EVENT_TRACKING_BACKENDS['segmentio']['OPTIONS']['processors'][0]['OPTIONS']['whitelist'].extend( + AUTH_TOKENS.get("EVENT_TRACKING_SEGMENTIO_EMIT_WHITELIST", [])) SUBDOMAIN_BRANDING = ENV_TOKENS.get('SUBDOMAIN_BRANDING', {}) VIRTUAL_UNIVERSITIES = ENV_TOKENS.get('VIRTUAL_UNIVERSITIES', []) diff --git a/cms/envs/common.py b/cms/envs/common.py index 445df80d30..5d19e4d538 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -776,19 +776,42 @@ TRACKING_IGNORE_URL_PATTERNS = [r'^/event', r'^/login', r'^/heartbeat'] EVENT_TRACKING_ENABLED = True EVENT_TRACKING_BACKENDS = { - 'logger': { - 'ENGINE': 'eventtracking.backends.logger.LoggerBackend', + 'tracking_logs': { + 'ENGINE': 'eventtracking.backends.routing.RoutingBackend', 'OPTIONS': { - 'name': 'tracking', - 'max_event_size': TRACK_MAX_EVENT, + 'backends': { + 'logger': { + 'ENGINE': 'eventtracking.backends.logger.LoggerBackend', + 'OPTIONS': { + 'name': 'tracking', + 'max_event_size': TRACK_MAX_EVENT, + } + } + }, + 'processors': [ + {'ENGINE': 'track.shim.LegacyFieldMappingProcessor'}, + {'ENGINE': 'track.shim.VideoEventProcessor'} + ] + } + }, + 'segmentio': { + 'ENGINE': 'eventtracking.backends.routing.RoutingBackend', + 'OPTIONS': { + 'backends': { + 'segment': {'ENGINE': 'eventtracking.backends.segment.SegmentBackend'} + }, + 'processors': [ + { + 'ENGINE': 'eventtracking.processors.whitelist.NameWhitelistProcessor', + 'OPTIONS': { + 'whitelist': [] + } + } + ] } } } -EVENT_TRACKING_PROCESSORS = [ - { - 'ENGINE': 'track.shim.LegacyFieldMappingProcessor' - } -] +EVENT_TRACKING_PROCESSORS = [] #### PASSWORD POLICY SETTINGS ##### diff --git a/common/djangoapps/track/shim.py b/common/djangoapps/track/shim.py index ddba730f9f..cef42fdba4 100644 --- a/common/djangoapps/track/shim.py +++ b/common/djangoapps/track/shim.py @@ -31,7 +31,10 @@ class LegacyFieldMappingProcessor(object): remove_shim_context(event) if 'data' in event: - event['event'] = event['data'] + if context.get('event_source', '') == 'browser' and isinstance(event['data'], dict): + event['event'] = json.dumps(event['data']) + else: + event['event'] = event['data'] del event['data'] else: event['event'] = {} @@ -103,7 +106,8 @@ class VideoEventProcessor(object): if name not in NAME_TO_EVENT_TYPE_MAP: return - # Convert edx.video.seeked to edx.video.positiion.changed + # Convert edx.video.seeked to edx.video.position.changed because edx.video.seeked was not intended to actually + # ever be emitted. if name == "edx.video.seeked": event['name'] = "edx.video.position.changed" diff --git a/common/djangoapps/track/tests/__init__.py b/common/djangoapps/track/tests/__init__.py index f394d143a2..3b99e5811c 100644 --- a/common/djangoapps/track/tests/__init__.py +++ b/common/djangoapps/track/tests/__init__.py @@ -31,22 +31,6 @@ class InMemoryBackend(object): self.events.append(event) -def unicode_flatten(tree): - """ - Test cases have funny issues where some strings are unicode, and - some are not. This does not cause test failures, but causes test - output diffs to show many more difference than actually occur in the - data. This will convert everything to a common form. - """ - if isinstance(tree, basestring): - return unicode(tree) - elif isinstance(tree, list): - return map(unicode_flatten, list) - elif isinstance(tree, dict): - return dict([(unicode_flatten(key), unicode_flatten(value)) for key, value in tree.iteritems()]) - return tree - - @freeze_time(FROZEN_TIME) @override_settings( EVENT_TRACKING_BACKENDS=IN_MEMORY_BACKEND_CONFIG @@ -64,6 +48,14 @@ class EventTrackingTestCase(TestCase): def setUp(self): super(EventTrackingTestCase, self).setUp() + self.recreate_tracker() + + def recreate_tracker(self): + """ + Re-initialize the tracking system using updated django settings. + + Use this if you make use of the @override_settings decorator to customize the tracker configuration. + """ self.tracker = DjangoTracker() tracker.register_tracker(self.tracker) @@ -83,7 +75,3 @@ class EventTrackingTestCase(TestCase): def assert_events_emitted(self): """Ensure at least one event has been emitted at this point in the test.""" self.assertGreaterEqual(len(self.backend.events), 1) - - def assertEqualUnicode(self, tree_a, tree_b): - """Like assertEqual, but give nicer errors for unicode vs. non-unicode""" - self.assertEqual(unicode_flatten(tree_a), unicode_flatten(tree_b)) diff --git a/common/djangoapps/track/tests/test_shim.py b/common/djangoapps/track/tests/test_shim.py index d28777f6bb..2f8a0fa67b 100644 --- a/common/djangoapps/track/tests/test_shim.py +++ b/common/djangoapps/track/tests/test_shim.py @@ -3,6 +3,7 @@ from mock import sentinel from django.test.utils import override_settings +from openedx.core.lib.tests.assertions.events import assert_events_equal from track.tests import EventTrackingTestCase, FROZEN_TIME @@ -13,12 +14,12 @@ LEGACY_SHIM_PROCESSOR = [ ] +@override_settings( + EVENT_TRACKING_PROCESSORS=LEGACY_SHIM_PROCESSOR, +) class LegacyFieldMappingProcessorTestCase(EventTrackingTestCase): """Ensure emitted events contain the fields legacy processors expect to find.""" - @override_settings( - EVENT_TRACKING_PROCESSORS=LEGACY_SHIM_PROCESSOR, - ) def test_event_field_mapping(self): data = {sentinel.key: sentinel.value} @@ -62,11 +63,8 @@ class LegacyFieldMappingProcessorTestCase(EventTrackingTestCase): 'page': None, 'session': sentinel.session, } - self.assertEqualUnicode(expected_event, emitted_event) + assert_events_equal(expected_event, emitted_event) - @override_settings( - EVENT_TRACKING_PROCESSORS=LEGACY_SHIM_PROCESSOR, - ) def test_missing_fields(self): self.tracker.emit(sentinel.name) @@ -88,4 +86,4 @@ class LegacyFieldMappingProcessorTestCase(EventTrackingTestCase): 'page': None, 'session': '', } - self.assertEqualUnicode(expected_event, emitted_event) + assert_events_equal(expected_event, emitted_event) diff --git a/common/djangoapps/track/views/__init__.py b/common/djangoapps/track/views/__init__.py index a7aaa5045f..17a0fa8ab8 100644 --- a/common/djangoapps/track/views/__init__.py +++ b/common/djangoapps/track/views/__init__.py @@ -1,4 +1,5 @@ import datetime +import json import pytz @@ -45,36 +46,28 @@ def user_track(request): GET or POST call should provide "event_type", "event", and "page" arguments. """ - try: # TODO: Do the same for many of the optional META parameters + try: username = request.user.username except: username = "anonymous" + name = _get_request_value(request, 'event_type') + data = _get_request_value(request, 'event', {}) page = _get_request_value(request, 'page') - with eventtracker.get_tracker().context('edx.course.browser', contexts.course_context_from_url(page)): - context = eventtracker.get_tracker().resolve_context() - event = { - "username": username, - "session": context.get('session', ''), - "ip": _get_request_header(request, 'REMOTE_ADDR'), - "referer": _get_request_header(request, 'HTTP_REFERER'), - "accept_language": _get_request_header(request, 'HTTP_ACCEPT_LANGUAGE'), - "event_source": "browser", - "event_type": _get_request_value(request, 'event_type'), - "event": _get_request_value(request, 'event'), - "agent": _get_request_header(request, 'HTTP_USER_AGENT'), - "page": page, - "time": datetime.datetime.utcnow(), - "host": _get_request_header(request, 'SERVER_NAME'), - "context": context, - } + if isinstance(data, basestring) and len(data) > 0: + try: + data = json.loads(data) + except ValueError: + pass - # Some duplicated fields are passed into event-tracking via the context by track.middleware. - # Remove them from the event here since they are captured elsewhere. - shim.remove_shim_context(event) + context_override = contexts.course_context_from_url(page) + context_override['username'] = username + context_override['event_source'] = 'browser' + context_override['page'] = page - log_event(event) + with eventtracker.get_tracker().context('edx.course.browser', context_override): + eventtracker.emit(name=name, data=data) return HttpResponse('success') diff --git a/common/djangoapps/track/views/tests/test_segmentio.py b/common/djangoapps/track/views/tests/test_segmentio.py index 09a1dd1220..590181e64e 100644 --- a/common/djangoapps/track/views/tests/test_segmentio.py +++ b/common/djangoapps/track/views/tests/test_segmentio.py @@ -10,6 +10,7 @@ from django.contrib.auth.models import User from django.test.client import RequestFactory from django.test.utils import override_settings +from openedx.core.lib.tests.assertions.events import assert_event_matches from track.middleware import TrackMiddleware from track.tests import EventTrackingTestCase from track.views import segmentio @@ -227,7 +228,7 @@ class SegmentIOTrackingTestCase(EventTrackingTestCase): finally: middleware.process_response(request, None) - self.assertEqualUnicode(self.get_event(), expected_event) + assert_event_matches(expected_event, self.get_event()) def test_invalid_course_id(self): request = self.create_request( @@ -331,6 +332,9 @@ class SegmentIOTrackingTestCase(EventTrackingTestCase): 'code': 'mobile' } if name == 'edx.video.loaded': + # We use the same expected payload for all of these types of events, but the load video event is the only + # one that is not actually expected to contain a "current time" field. So we remove it from the expected + # event here. del input_payload['current_time'] request = self.create_request( @@ -355,7 +359,7 @@ class SegmentIOTrackingTestCase(EventTrackingTestCase): response = segmentio.segmentio_event(request) self.assertEquals(response.status_code, 200) - expected_event_without_payload = { + expected_event = { 'accept_language': '', 'referer': '', 'username': str(sentinel.username), @@ -389,22 +393,22 @@ class SegmentIOTrackingTestCase(EventTrackingTestCase): }, 'received_at': datetime.strptime("2014-08-27T16:33:39.100Z", "%Y-%m-%dT%H:%M:%S.%fZ"), }, - } - expected_payload = { - 'currentTime': 132.134456, - 'id': 'i4x-foo-bar-baz-some_module', - 'code': 'mobile' + 'event': { + 'currentTime': 132.134456, + 'id': 'i4x-foo-bar-baz-some_module', + 'code': 'mobile' + } } if name == 'edx.video.loaded': - del expected_payload['currentTime'] + # We use the same expected payload for all of these types of events, but the load video event is the + # only one that is not actually expected to contain a "current time" field. So we remove it from the + # expected event here. + del expected_event['event']['currentTime'] finally: middleware.process_response(request, None) - actual_event = dict(self.get_event()) - payload = json.loads(actual_event.pop('event')) - - self.assertEqualUnicode(actual_event, expected_event_without_payload) - self.assertEqualUnicode(payload, expected_payload) + actual_event = self.get_event() + assert_event_matches(expected_event, actual_event) @data( # Verify positive slide case. Verify slide to onSlideSeek. Verify edx.video.seeked emitted from iOS v1.0.02 is changed to edx.video.position.changed. @@ -479,7 +483,7 @@ class SegmentIOTrackingTestCase(EventTrackingTestCase): response = segmentio.segmentio_event(request) self.assertEquals(response.status_code, 200) - expected_event_without_payload = { + expected_event = { 'accept_language': '', 'referer': '', 'username': str(sentinel.username), @@ -513,19 +517,17 @@ class SegmentIOTrackingTestCase(EventTrackingTestCase): }, 'received_at': datetime.strptime("2014-08-27T16:33:39.100Z", "%Y-%m-%dT%H:%M:%S.%fZ"), }, - } - expected_payload = { - "code": "mobile", - "new_time": 89.699177437, - "old_time": 119.699177437, - "type": expected_seek_type, - "requested_skip_interval": expected_skip_interval, - 'id': 'i4x-foo-bar-baz-some_module', + 'event': { + "code": "mobile", + "new_time": 89.699177437, + "old_time": 119.699177437, + "type": expected_seek_type, + "requested_skip_interval": expected_skip_interval, + 'id': 'i4x-foo-bar-baz-some_module', + } } finally: middleware.process_response(request, None) - actual_event = dict(self.get_event()) - payload = json.loads(actual_event.pop('event')) - self.assertEqualUnicode(actual_event, expected_event_without_payload) - self.assertEqualUnicode(payload, expected_payload) + actual_event = self.get_event() + assert_event_matches(expected_event, actual_event) diff --git a/common/djangoapps/track/views/tests/test_views.py b/common/djangoapps/track/views/tests/test_views.py index d8d5a81007..18c684fda5 100644 --- a/common/djangoapps/track/views/tests/test_views.py +++ b/common/djangoapps/track/views/tests/test_views.py @@ -1,22 +1,26 @@ # pylint: disable=missing-docstring,maybe-no-member -from track import views -from track.middleware import TrackMiddleware from mock import patch, sentinel -from freezegun import freeze_time +from django.contrib.auth.models import User from django.test import TestCase from django.test.client import RequestFactory +from django.test.utils import override_settings from eventtracking import tracker +from track import views +from track.middleware import TrackMiddleware +from track.tests import EventTrackingTestCase, FROZEN_TIME +from openedx.core.lib.tests.assertions.events import assert_event_matches from datetime import datetime -expected_time = datetime(2013, 10, 3, 8, 24, 55) -class TestTrackViews(TestCase): +class TestTrackViews(EventTrackingTestCase): def setUp(self): + super(TestTrackViews, self).setUp() + self.request_factory = RequestFactory() patcher = patch('track.views.tracker') @@ -30,100 +34,127 @@ class TestTrackViews(TestCase): sentinel.key: sentinel.value } - @freeze_time(expected_time) def test_user_track(self): request = self.request_factory.get('/event', { 'page': self.url_with_course, 'event_type': sentinel.event_type, - 'event': {} + 'event': '{}' }) - with tracker.get_tracker().context('edx.request', {'session': sentinel.session}): - views.user_track(request) + views.user_track(request) + + actual_event = self.get_event() expected_event = { - 'accept_language': '', - 'referer': '', - 'username': 'anonymous', - 'session': sentinel.session, - 'ip': '127.0.0.1', - 'event_source': 'browser', - 'event_type': str(sentinel.event_type), - 'event': '{}', - 'agent': '', - 'page': self.url_with_course, - 'time': expected_time, - 'host': 'testserver', 'context': { 'course_id': 'foo/bar/baz', 'org_id': 'foo', + 'event_source': 'browser', + 'page': self.url_with_course, + 'username': 'anonymous' }, + 'data': {}, + 'timestamp': FROZEN_TIME, + 'name': str(sentinel.event_type) } - self.mock_tracker.send.assert_called_once_with(expected_event) + assert_event_matches(expected_event, actual_event) - @freeze_time(expected_time) def test_user_track_with_missing_values(self): request = self.request_factory.get('/event') - with tracker.get_tracker().context('edx.request', {'session': sentinel.session}): - views.user_track(request) + views.user_track(request) + + actual_event = self.get_event() expected_event = { - 'accept_language': '', - 'referer': '', - 'username': 'anonymous', - 'session': sentinel.session, - 'ip': '127.0.0.1', - 'event_source': 'browser', - 'event_type': '', - 'event': '', - 'agent': '', - 'page': '', - 'time': expected_time, - 'host': 'testserver', 'context': { 'course_id': '', 'org_id': '', + 'event_source': 'browser', + 'page': '', + 'username': 'anonymous' }, + 'data': {}, + 'timestamp': FROZEN_TIME, + 'name': 'unknown' } - self.mock_tracker.send.assert_called_once_with(expected_event) + assert_event_matches(expected_event, actual_event) - @freeze_time(expected_time) - def test_user_track_with_middleware(self): - middleware = TrackMiddleware() + views.user_track(request) + + def test_user_track_with_empty_event(self): request = self.request_factory.get('/event', { 'page': self.url_with_course, 'event_type': sentinel.event_type, - 'event': {} + 'event': '' }) + + views.user_track(request) + + actual_event = self.get_event() + expected_event = { + 'context': { + 'course_id': 'foo/bar/baz', + 'org_id': 'foo', + 'event_source': 'browser', + 'page': self.url_with_course, + 'username': 'anonymous' + }, + 'data': {}, + 'timestamp': FROZEN_TIME, + 'name': str(sentinel.event_type) + } + assert_event_matches(expected_event, actual_event) + + @override_settings( + EVENT_TRACKING_PROCESSORS=[{'ENGINE': 'track.shim.LegacyFieldMappingProcessor'}], + ) + def test_user_track_with_middleware_and_processors(self): + self.recreate_tracker() + + middleware = TrackMiddleware() + payload = '{"foo": "bar"}' + user_id = 1 + request = self.request_factory.get('/event', { + 'page': self.url_with_course, + 'event_type': sentinel.event_type, + 'event': payload + }) + request.user = User.objects.create(pk=user_id, username=str(sentinel.username)) + request.META['REMOTE_ADDR'] = '10.0.0.1' + request.META['HTTP_REFERER'] = str(sentinel.referer) + request.META['HTTP_ACCEPT_LANGUAGE'] = str(sentinel.accept_language) + request.META['HTTP_USER_AGENT'] = str(sentinel.user_agent) + request.META['SERVER_NAME'] = 'testserver2' middleware.process_request(request) try: views.user_track(request) expected_event = { - 'accept_language': '', - 'referer': '', - 'username': 'anonymous', + 'accept_language': str(sentinel.accept_language), + 'referer': str(sentinel.referer), + 'username': str(sentinel.username), 'session': '', - 'ip': '127.0.0.1', + 'ip': '10.0.0.1', 'event_source': 'browser', 'event_type': str(sentinel.event_type), - 'event': '{}', - 'agent': '', + 'name': str(sentinel.event_type), + 'event': payload, + 'agent': str(sentinel.user_agent), 'page': self.url_with_course, - 'time': expected_time, - 'host': 'testserver', + 'time': FROZEN_TIME, + 'host': 'testserver2', 'context': { 'course_id': 'foo/bar/baz', 'org_id': 'foo', - 'user_id': '', + 'user_id': user_id, 'path': u'/event' }, } finally: middleware.process_response(request, None) - self.mock_tracker.send.assert_called_once_with(expected_event) + actual_event = self.get_event() + assert_event_matches(expected_event, actual_event) - @freeze_time(expected_time) def test_server_track(self): request = self.request_factory.get(self.path_with_course) views.server_track(request, str(sentinel.event_type), '{}') @@ -138,13 +169,17 @@ class TestTrackViews(TestCase): 'event': '{}', 'agent': '', 'page': None, - 'time': expected_time, + 'time': FROZEN_TIME, 'host': 'testserver', 'context': {}, } - self.mock_tracker.send.assert_called_once_with(expected_event) + self.assert_mock_tracker_call_matches(expected_event) + + def assert_mock_tracker_call_matches(self, expected_event): + self.assertEqual(len(self.mock_tracker.send.mock_calls), 1) + actual_event = self.mock_tracker.send.mock_calls[0][1][0] + assert_event_matches(expected_event, actual_event) - @freeze_time(expected_time) def test_server_track_with_middleware(self): middleware = TrackMiddleware() request = self.request_factory.get(self.path_with_course) @@ -164,7 +199,7 @@ class TestTrackViews(TestCase): 'event': '{}', 'agent': '', 'page': None, - 'time': expected_time, + 'time': FROZEN_TIME, 'host': 'testserver', 'context': { 'user_id': '', @@ -176,9 +211,8 @@ class TestTrackViews(TestCase): finally: middleware.process_response(request, None) - self.mock_tracker.send.assert_called_once_with(expected_event) + self.assert_mock_tracker_call_matches(expected_event) - @freeze_time(expected_time) def test_server_track_with_middleware_and_google_analytics_cookie(self): middleware = TrackMiddleware() request = self.request_factory.get(self.path_with_course) @@ -199,7 +233,7 @@ class TestTrackViews(TestCase): 'event': '{}', 'agent': '', 'page': None, - 'time': expected_time, + 'time': FROZEN_TIME, 'host': 'testserver', 'context': { 'user_id': '', @@ -211,9 +245,8 @@ class TestTrackViews(TestCase): finally: middleware.process_response(request, None) - self.mock_tracker.send.assert_called_once_with(expected_event) + self.assert_mock_tracker_call_matches(expected_event) - @freeze_time(expected_time) def test_server_track_with_no_request(self): request = None views.server_track(request, str(sentinel.event_type), '{}') @@ -228,13 +261,12 @@ class TestTrackViews(TestCase): 'event': '{}', 'agent': '', 'page': None, - 'time': expected_time, + 'time': FROZEN_TIME, 'host': '', 'context': {}, } - self.mock_tracker.send.assert_called_once_with(expected_event) + self.assert_mock_tracker_call_matches(expected_event) - @freeze_time(expected_time) def test_task_track(self): request_info = { 'accept_language': '', @@ -261,11 +293,11 @@ class TestTrackViews(TestCase): 'event': expected_event_data, 'agent': 'agent', 'page': None, - 'time': expected_time, + 'time': FROZEN_TIME, 'host': 'testserver', 'context': { 'course_id': '', 'org_id': '' }, } - self.mock_tracker.send.assert_called_once_with(expected_event) + self.assert_mock_tracker_call_matches(expected_event) diff --git a/common/test/acceptance/fixtures/discussion.py b/common/test/acceptance/fixtures/discussion.py index cec8aa6003..38b20c768b 100644 --- a/common/test/acceptance/fixtures/discussion.py +++ b/common/test/acceptance/fixtures/discussion.py @@ -28,6 +28,13 @@ class ContentFactory(factory.Factory): closed = False votes = {"up_count": 0} + @classmethod + def _adjust_kwargs(cls, **kwargs): + # The discussion code assumes that user_id is a string. This ensures that it always will be. + if 'user_id' in kwargs: + kwargs['user_id'] = str(kwargs['user_id']) + return kwargs + class Thread(ContentFactory): thread_type = "discussion" diff --git a/common/test/acceptance/pages/lms/auto_auth.py b/common/test/acceptance/pages/lms/auto_auth.py index cdc60771e1..d9113719e1 100644 --- a/common/test/acceptance/pages/lms/auto_auth.py +++ b/common/test/acceptance/pages/lms/auto_auth.py @@ -4,7 +4,7 @@ Auto-auth page (used to automatically log in during testing). import re import urllib -from bok_choy.page_object import PageObject +from bok_choy.page_object import PageObject, unguarded from . import AUTH_BASE_URL @@ -15,6 +15,8 @@ class AutoAuthPage(PageObject): this url will create a user and log them in. """ + CONTENT_REGEX = r'.+? user (?P\S+) \((?P.+?)\) with password \S+ and user_id (?P\d+)$' + def __init__(self, browser, username=None, email=None, password=None, staff=None, course_id=None, roles=None): """ Auto-auth is an end-point for HTTP GET requests. @@ -30,6 +32,9 @@ class AutoAuthPage(PageObject): """ super(AutoAuthPage, self).__init__(browser) + # This will eventually hold the details about the user account + self._user_info = None + # Create query string parameters if provided self._params = {} @@ -65,14 +70,31 @@ class AutoAuthPage(PageObject): return url def is_browser_on_page(self): + return True if self.get_user_info() is not None else False + + @unguarded + def get_user_info(self): + """Parse the auto auth page body to extract relevant details about the user that was logged in.""" message = self.q(css='BODY').text[0] - match = re.search(r'Logged in user ([^$]+) with password ([^$]+) and user_id ([^$]+)$', message) - return True if match else False + match = re.match(self.CONTENT_REGEX, message) + if not match: + return None + else: + user_info = match.groupdict() + user_info['user_id'] = int(user_info['user_id']) + return user_info + + @property + def user_info(self): + """A dictionary containing details about the user account.""" + if self._user_info is None: + user_info = self.get_user_info() + if user_info is not None: + self._user_info = self.get_user_info() + return self._user_info def get_user_id(self): """ Finds and returns the user_id """ - message = self.q(css='BODY').text[0].strip() - match = re.search(r' user_id ([^$]+)$', message) - return match.groups()[0] if match else None + return self.user_info['user_id'] diff --git a/common/test/acceptance/tests/helpers.py b/common/test/acceptance/tests/helpers.py index 14bcacf79c..2ccb8f0509 100644 --- a/common/test/acceptance/tests/helpers.py +++ b/common/test/acceptance/tests/helpers.py @@ -1,18 +1,24 @@ """ Test helper functions and base classes. """ +import inspect import json import unittest import functools +import operator +import pprint import requests import os +import urlparse +from contextlib import contextmanager from datetime import datetime from path import path from bok_choy.javascript import js_defined from bok_choy.web_app_test import WebAppTest -from bok_choy.promise import EmptyPromise +from bok_choy.promise import EmptyPromise, Promise from opaque_keys.edx.locator import CourseLocator -from pymongo import MongoClient +from pymongo import MongoClient, ASCENDING +from openedx.core.lib.tests.assertions.events import assert_event_matches, is_matching_event, EventMatchTolerates from xmodule.partitions.partitions import UserPartition from xmodule.partitions.tests.test_partitions import MockUserPartitionScheme from selenium.webdriver.support.select import Select @@ -20,6 +26,12 @@ from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC +from ..pages.common import BASE_URL + + +MAX_EVENTS_IN_FAILURE_OUTPUT = 20 + + def skip_if_browser(browser): """ Method decorator that skips a test if browser is `browser` @@ -279,81 +291,257 @@ class EventsTestMixin(object): self.event_collection = MongoClient()["test"]["events"] self.reset_event_tracking() - def assert_event_emitted_num_times(self, event_name, event_time, event_user_id, num_times_emitted, **kwargs): - """ - Tests the number of times a particular event was emitted. - - Extra kwargs get passed to the mongo query in the form: "event.: value". - - :param event_name: Expected event name (e.g., "edx.course.enrollment.activated") - :param event_time: Latest expected time, after which the event would fire (e.g., the beginning of the test case) - :param event_user_id: user_id expected in the event - :param num_times_emitted: number of times the event is expected to appear since the event_time - """ - find_kwargs = { - "name": event_name, - "time": {"$gt": event_time}, - "event.user_id": int(event_user_id), - } - find_kwargs.update({"event.{}".format(key): value for key, value in kwargs.items()}) - matching_events = self.event_collection.find(find_kwargs) - self.assertEqual(matching_events.count(), num_times_emitted, '\n'.join(str(event) for event in matching_events)) - def reset_event_tracking(self): - """ - Resets all event tracking so that previously captured events are removed. - """ + """Drop any events that have been collected thus far and start collecting again from scratch.""" self.event_collection.drop() self.start_time = datetime.now() - def get_matching_events(self, username, event_type): + @contextmanager + def capture_events(self, event_filter=None, number_of_matches=1, captured_events=None): """ - Returns a cursor for the matching browser events related emitted for the specified username. - """ - return self.event_collection.find({ - "username": username, - "event_type": event_type, - "time": {"$gt": self.start_time}, - }) + Context manager that captures all events emitted while executing a particular block. - def verify_events_of_type(self, username, event_type, expected_events, expected_referers=None): - """Verify that the expected events of a given type were logged. - Args: - username (str): The name of the user for which events will be tested. - event_type (str): The type of event to be verified. - expected_events (list): A list of dicts representing the events that should - have been fired. - expected_referers (list): A list of strings representing the referers for each event - that should been fired (optional). If present, the actual referers compared - with this list, checking that the expected_referers are the suffixes of - actual_referers. For example, if one event is expected, specifying ["/account/settings"] - will verify that the referer for the single event ends with "/account/settings". + All captured events are stored in the list referenced by `captured_events`. Note that this list is appended to + *in place*. The events will be appended to the list in the order they are emitted. + + The `event_filter` is expected to be a callable that allows you to filter the event stream and select particular + events of interest. A dictionary `event_filter` is also supported, which simply indicates that the event should + match that provided expectation. + + `number_of_matches` tells this context manager when enough events have been found and it can move on. The + context manager will not exit until this many events have passed the filter. If not enough events are found + before a timeout expires, then this will raise a `BrokenPromise` error. Note that this simply states that + *at least* this many events have been emitted, so `number_of_matches` is simply a lower bound for the size of + `captured_events`. """ - EmptyPromise( - lambda: self.get_matching_events(username, event_type).count() >= len(expected_events), - "Waiting for the minimum number of events of type {type} to have been recorded".format(type=event_type) + start_time = datetime.utcnow() + + yield + + events = self.wait_for_events( + start_time=start_time, event_filter=event_filter, number_of_matches=number_of_matches) + + if captured_events is not None and hasattr(captured_events, 'append') and callable(captured_events.append): + for event in events: + captured_events.append(event) + + @contextmanager + def assert_events_match_during(self, event_filter=None, expected_events=None): + """ + Context manager that ensures that events matching the `event_filter` and `expected_events` are emitted. + + This context manager will filter out the event stream using the `event_filter` and wait for + `len(expected_events)` to match the filter. + + It will then compare the events in order with their counterpart in `expected_events` to ensure they match the + more detailed assertion. + + Typically `event_filter` will be an `event_type` filter and the `expected_events` list will contain more + detailed assertions. + """ + captured_events = [] + with self.capture_events(event_filter, len(expected_events), captured_events): + yield + + self.assert_events_match(expected_events, captured_events) + + def wait_for_events(self, start_time=None, event_filter=None, number_of_matches=1, timeout=None): + """ + Wait for `number_of_matches` events to pass the `event_filter`. + + By default, this will look at all events that have been emitted since the beginning of the setup of this mixin. + A custom `start_time` can be specified which will limit the events searched to only those emitted after that + time. + + The `event_filter` is expected to be a callable that allows you to filter the event stream and select particular + events of interest. A dictionary `event_filter` is also supported, which simply indicates that the event should + match that provided expectation. + + `number_of_matches` lets us know when enough events have been found and it can move on. The function will not + return until this many events have passed the filter. If not enough events are found before a timeout expires, + then this will raise a `BrokenPromise` error. Note that this simply states that *at least* this many events have + been emitted, so `number_of_matches` is simply a lower bound for the size of `captured_events`. + + Specifying a custom `timeout` can allow you to extend the default 30 second timeout if necessary. + """ + if start_time is None: + start_time = self.start_time + + if timeout is None: + timeout = 30 + + def check_for_matching_events(): + """Gather any events that have been emitted since `start_time`""" + return self.matching_events_were_emitted( + start_time=start_time, + event_filter=event_filter, + number_of_matches=number_of_matches + ) + + return Promise( + check_for_matching_events, + # This is a bit of a hack, Promise calls str(description), so I set the description to an object with a + # custom __str__ and have it do some intelligent stuff to generate a helpful error message. + CollectedEventsDescription( + 'Waiting for {number_of_matches} events to match the filter:\n{event_filter}'.format( + number_of_matches=number_of_matches, + event_filter=self.event_filter_to_descriptive_string(event_filter), + ), + functools.partial(self.get_matching_events_from_time, start_time=start_time, event_filter={}) + ), + timeout=timeout ).fulfill() - # Verify that the correct events were fired - cursor = self.get_matching_events(username, event_type) - actual_events = [] - actual_referers = [] - for __ in range(0, cursor.count()): - emitted_data = cursor.next() - event = emitted_data["event"] - if emitted_data["event_source"] == "browser": - event = json.loads(event) - actual_events.append(event) - actual_referers.append(emitted_data["referer"]) - self.assertEqual(expected_events, actual_events) - if expected_referers is not None: - self.assertEqual(len(expected_referers), len(actual_referers), "Number of expected referers is incorrect") - for index, actual_referer in enumerate(actual_referers): - self.assertTrue( - actual_referer.endswith(expected_referers[index]), - "Refer '{0}' does not have correct suffix, '{1}'.".format(actual_referer, expected_referers[index]) + def matching_events_were_emitted(self, start_time=None, event_filter=None, number_of_matches=1): + """Return True if enough events have been emitted that pass the `event_filter` since `start_time`.""" + matching_events = self.get_matching_events_from_time(start_time=start_time, event_filter=event_filter) + return len(matching_events) >= number_of_matches, matching_events + + def get_matching_events_from_time(self, start_time=None, event_filter=None): + """ + Return a list of events that pass the `event_filter` and were emitted after `start_time`. + + This function is used internally by most of the other assertions and convenience methods in this class. + + The `event_filter` is expected to be a callable that allows you to filter the event stream and select particular + events of interest. A dictionary `event_filter` is also supported, which simply indicates that the event should + match that provided expectation. + """ + if start_time is None: + start_time = self.start_time + + if isinstance(event_filter, dict): + event_filter = functools.partial(is_matching_event, event_filter) + elif not callable(event_filter): + raise ValueError( + 'event_filter must either be a dict or a callable function with as single "event" parameter that ' + 'returns a boolean value.' + ) + + matching_events = [] + cursor = self.event_collection.find( + { + "time": { + "$gte": start_time + } + } + ).sort("time", ASCENDING) + for event in cursor: + matches = False + try: + # Mongo automatically assigns an _id to all events inserted into it. We strip it out here, since + # we don't care about it. + del event['_id'] + if event_filter is not None: + # Typically we will be grabbing all events of a particular type, however, you can use arbitrary + # logic to identify the events that are of interest. + matches = event_filter(event) + except AssertionError: + # allow the filters to use "assert" to filter out events + continue + else: + if matches is None or matches: + matching_events.append(event) + return matching_events + + def assert_matching_events_were_emitted(self, start_time=None, event_filter=None, number_of_matches=1): + """Assert that at least `number_of_matches` events have passed the filter since `start_time`.""" + description = CollectedEventsDescription( + 'Not enough events match the filter:\n' + self.event_filter_to_descriptive_string(event_filter), + functools.partial(self.get_matching_events_from_time, start_time=start_time, event_filter={}) + ) + + self.assertTrue( + self.matching_events_were_emitted( + start_time=start_time, event_filter=event_filter, number_of_matches=number_of_matches + ), + description + ) + + def assert_no_matching_events_were_emitted(self, event_filter, start_time=None): + """Assert that no events have passed the filter since `start_time`.""" + matching_events = self.get_matching_events_from_time(start_time=start_time, event_filter=event_filter) + + description = CollectedEventsDescription( + 'Events unexpected matched the filter:\n' + self.event_filter_to_descriptive_string(event_filter), + lambda: matching_events + ) + + self.assertEquals(len(matching_events), 0, description) + + def assert_events_match(self, expected_events, actual_events): + """ + Assert that each item in the expected events sequence matches its counterpart at the same index in the actual + events sequence. + """ + for expected_event, actual_event in zip(expected_events, actual_events): + assert_event_matches( + expected_event, + actual_event, + tolerate=EventMatchTolerates.lenient() + ) + + def relative_path_to_absolute_uri(self, relative_path): + """Return an aboslute URI given a relative path taking into account the test context.""" + return urlparse.urljoin(BASE_URL, relative_path) + + def event_filter_to_descriptive_string(self, event_filter): + """Find the source code of the callable or pretty-print the dictionary""" + message = '' + if callable(event_filter): + file_name = '(unknown)' + try: + file_name = inspect.getsourcefile(event_filter) + except TypeError: + pass + + try: + list_of_source_lines, line_no = inspect.getsourcelines(event_filter) + except IOError: + pass + else: + message = '{file_name}:{line_no}\n{hr}\n{event_filter}\n{hr}'.format( + event_filter=''.join(list_of_source_lines).rstrip(), + file_name=file_name, + line_no=line_no, + hr='-' * 20, ) + if not message: + message = '{hr}\n{event_filter}\n{hr}'.format( + event_filter=pprint.pformat(event_filter), + hr='-' * 20, + ) + + return message + + +class CollectedEventsDescription(object): + """ + Produce a clear error message when tests fail. + + This class calls the provided `get_events_func` when converted to a string, and pretty prints the returned events. + """ + + def __init__(self, description, get_events_func): + self.description = description + self.get_events_func = get_events_func + + def __str__(self): + message_lines = [ + self.description, + 'Events:' + ] + events = self.get_events_func() + events.sort(key=operator.itemgetter('time'), reverse=True) + for event in events[:MAX_EVENTS_IN_FAILURE_OUTPUT]: + message_lines.append(pprint.pformat(event)) + if len(events) > MAX_EVENTS_IN_FAILURE_OUTPUT: + message_lines.append( + 'Too many events to display, the remaining events were omitted. Run locally to diagnose.') + + return '\n\n'.join(message_lines) + class UniqueCourseTest(WebAppTest): """ diff --git a/common/test/acceptance/tests/lms/test_account_settings.py b/common/test/acceptance/tests/lms/test_account_settings.py index 82845d697f..1881e29006 100644 --- a/common/test/acceptance/tests/lms/test_account_settings.py +++ b/common/test/acceptance/tests/lms/test_account_settings.py @@ -33,30 +33,49 @@ class AccountSettingsTestMixin(EventsTestMixin, WebAppTest): user_id = auto_auth_page.get_user_id() return username, user_id - def assert_event_emitted_num_times(self, user_id, setting, num_times): - """ - Verify a particular user settings change event was emitted a certain - number of times. - """ - # pylint disable=no-member - super(AccountSettingsTestMixin, self).assert_event_emitted_num_times( - self.USER_SETTINGS_CHANGED_EVENT_NAME, self.start_time, user_id, num_times, setting=setting - ) + def settings_changed_event_filter(self, event): + """Filter out any events that are not "settings changed" events.""" + return event['event_type'] == self.USER_SETTINGS_CHANGED_EVENT_NAME - def verify_settings_changed_events(self, username, user_id, events, table=None): - """ - Verify a particular set of account settings change events were fired. - """ - expected_referers = [self.ACCOUNT_SETTINGS_REFERER] * len(events) - for event in events: - event[u"user_id"] = long(user_id) - event[u"table"] = u"auth_userprofile" if table is None else table - event[u"truncated"] = [] + def expected_settings_changed_event(self, setting, old, new, table=None): + """A dictionary representing the expected fields in a "settings changed" event.""" + return { + 'username': self.username, + 'referer': self.get_settings_page_url(), + 'event': { + 'user_id': self.user_id, + 'setting': setting, + 'old': old, + 'new': new, + 'truncated': [], + 'table': table or 'auth_userprofile' + } + } - self.verify_events_of_type( - username, self.USER_SETTINGS_CHANGED_EVENT_NAME, events, - expected_referers=expected_referers - ) + def settings_change_initiated_event_filter(self, event): + """Filter out any events that are not "settings change initiated" events.""" + return event['event_type'] == self.CHANGE_INITIATED_EVENT_NAME + + def expected_settings_change_initiated_event(self, setting, old, new, username=None, user_id=None): + """A dictionary representing the expected fields in a "settings change initiated" event.""" + return { + 'username': username or self.username, + 'referer': self.get_settings_page_url(), + 'event': { + 'user_id': user_id or self.user_id, + 'setting': setting, + 'old': old, + 'new': new, + } + } + + def get_settings_page_url(self): + """The absolute URL of the account settings page given the test context.""" + return self.relative_path_to_absolute_uri(self.ACCOUNT_SETTINGS_REFERER) + + def assert_no_setting_changed_event(self): + """Assert no setting changed event has been emitted thus far.""" + self.assert_no_matching_events_were_emitted({'event_type': self.USER_SETTINGS_CHANGED_EVENT_NAME}) @attr('shard_5') @@ -114,14 +133,20 @@ class AccountSettingsPageTest(AccountSettingsTestMixin, WebAppTest): And I visit my account settings page Then a page view analytics event should be recorded """ - self.verify_events_of_type( - self.username, - u"edx.user.settings.viewed", - [{ - u"user_id": long(self.user_id), - u"page": u"account", - u"visibility": None, - }] + + actual_events = self.wait_for_events( + event_filter={'event_type': 'edx.user.settings.viewed'}, number_of_matches=1) + self.assert_events_match( + [ + { + 'event': { + 'user_id': self.user_id, + 'page': 'account', + 'visibility': None + } + } + ], + actual_events ) def test_all_sections_and_fields_are_present(self): @@ -237,20 +262,13 @@ class AccountSettingsPageTest(AccountSettingsTestMixin, WebAppTest): [u'another name', self.username], ) - self.verify_settings_changed_events( - self.username, self.user_id, + actual_events = self.wait_for_events(event_filter=self.settings_changed_event_filter, number_of_matches=2) + self.assert_events_match( [ - { - u"setting": u"name", - u"old": self.username, - u"new": u"another name", - }, - { - u"setting": u"name", - u"old": u'another name', - u"new": self.username, - } - ] + self.expected_settings_changed_event('name', self.username, 'another name'), + self.expected_settings_changed_event('name', 'another name', self.username), + ], + actual_events ) def test_email_field(self): @@ -270,28 +288,21 @@ class AccountSettingsPageTest(AccountSettingsTestMixin, WebAppTest): assert_after_reload=False ) - self.verify_events_of_type( - username, - self.CHANGE_INITIATED_EVENT_NAME, + actual_events = self.wait_for_events( + event_filter=self.settings_change_initiated_event_filter, number_of_matches=2) + self.assert_events_match( [ - { - u"user_id": long(user_id), - u"setting": u"email", - u"old": email, - u"new": u'me@here.com' - }, - { - u"user_id": long(user_id), - u"setting": u"email", - u"old": email, # NOTE the first email change was never confirmed, so old has not changed. - u"new": u'you@there.com' - } + self.expected_settings_change_initiated_event( + 'email', email, 'me@here.com', username=username, user_id=user_id), + # NOTE the first email change was never confirmed, so old has not changed. + self.expected_settings_change_initiated_event( + 'email', email, 'you@there.com', username=username, user_id=user_id), ], - [self.ACCOUNT_SETTINGS_REFERER, self.ACCOUNT_SETTINGS_REFERER] + actual_events ) # Email is not saved until user confirms, so no events should have been # emitted. - self.assert_event_emitted_num_times(user_id, 'email', 0) + self.assert_no_setting_changed_event() def test_password_field(self): """ @@ -304,20 +315,11 @@ class AccountSettingsPageTest(AccountSettingsTestMixin, WebAppTest): success_message='Click the link in the message to reset your password.', ) - self.verify_events_of_type( - self.username, - self.CHANGE_INITIATED_EVENT_NAME, - [{ - u"user_id": int(self.user_id), - u"setting": "password", - u"old": None, - u"new": None - }], - [self.ACCOUNT_SETTINGS_REFERER] - ) + event_filter = self.expected_settings_change_initiated_event('password', None, None) + self.wait_for_events(event_filter=event_filter, number_of_matches=1) # Like email, since the user has not confirmed their password change, # the field has not yet changed, so no events will have been emitted. - self.assert_event_emitted_num_times(self.user_id, 'password', 0) + self.assert_no_setting_changed_event() @skip( 'On bokchoy test servers, language changes take a few reloads to fully realize ' @@ -345,20 +347,14 @@ class AccountSettingsPageTest(AccountSettingsTestMixin, WebAppTest): u'', [u'Bachelor\'s degree', u''], ) - self.verify_settings_changed_events( - self.username, self.user_id, + + actual_events = self.wait_for_events(event_filter=self.settings_changed_event_filter, number_of_matches=2) + self.assert_events_match( [ - { - u"setting": u"level_of_education", - u"old": None, - u"new": u'b', - }, - { - u"setting": u"level_of_education", - u"old": u'b', - u"new": None, - } - ] + self.expected_settings_changed_event('level_of_education', None, 'b'), + self.expected_settings_changed_event('level_of_education', 'b', None), + ], + actual_events ) def test_gender_field(self): @@ -371,20 +367,14 @@ class AccountSettingsPageTest(AccountSettingsTestMixin, WebAppTest): u'', [u'Female', u''], ) - self.verify_settings_changed_events( - self.username, self.user_id, + + actual_events = self.wait_for_events(event_filter=self.settings_changed_event_filter, number_of_matches=2) + self.assert_events_match( [ - { - u"setting": u"gender", - u"old": None, - u"new": u'f', - }, - { - u"setting": u"gender", - u"old": u'f', - u"new": None, - } - ] + self.expected_settings_changed_event('gender', None, 'f'), + self.expected_settings_changed_event('gender', 'f', None), + ], + actual_events ) def test_year_of_birth_field(self): @@ -393,28 +383,18 @@ class AccountSettingsPageTest(AccountSettingsTestMixin, WebAppTest): """ # Note that when we clear the year_of_birth here we're firing an event. self.assertEqual(self.account_settings_page.value_for_dropdown_field('year_of_birth', ''), '') - self.reset_event_tracking() - self._test_dropdown_field( - u'year_of_birth', - u'Year of Birth', - u'', - [u'1980', u''], - ) - self.verify_settings_changed_events( - self.username, self.user_id, - [ - { - u"setting": u"year_of_birth", - u"old": None, - u"new": 1980L, - }, - { - u"setting": u"year_of_birth", - u"old": 1980L, - u"new": None, - } - ] - ) + + expected_events = [ + self.expected_settings_changed_event('year_of_birth', None, 1980), + self.expected_settings_changed_event('year_of_birth', 1980, None), + ] + with self.assert_events_match_during(self.settings_changed_event_filter, expected_events): + self._test_dropdown_field( + u'year_of_birth', + u'Year of Birth', + u'', + [u'1980', u''], + ) def test_country_field(self): """ @@ -438,21 +418,15 @@ class AccountSettingsPageTest(AccountSettingsTestMixin, WebAppTest): [u'Pushto', u''], ) - self.verify_settings_changed_events( - self.username, self.user_id, + actual_events = self.wait_for_events(event_filter=self.settings_changed_event_filter, number_of_matches=2) + self.assert_events_match( [ - { - u"setting": u"language_proficiencies", - u"old": [], - u"new": [{u"code": u"ps"}], - }, - { - u"setting": u"language_proficiencies", - u"old": [{u"code": u"ps"}], - u"new": [], - } + self.expected_settings_changed_event( + 'language_proficiencies', [], [{'code': 'ps'}], table='student_languageproficiency'), + self.expected_settings_changed_event( + 'language_proficiencies', [{'code': 'ps'}], [], table='student_languageproficiency'), ], - table=u"student_languageproficiency" + actual_events ) def test_connected_accounts(self): diff --git a/common/test/acceptance/tests/lms/test_learner_profile.py b/common/test/acceptance/tests/lms/test_learner_profile.py index 7c0552372f..75725201da 100644 --- a/common/test/acceptance/tests/lms/test_learner_profile.py +++ b/common/test/acceptance/tests/lms/test_learner_profile.py @@ -2,6 +2,8 @@ """ End-to-end tests for Student's Profile Page. """ +from contextlib import contextmanager + from datetime import datetime from bok_choy.web_app_test import WebAppTest from nose.plugins.attrib import attr @@ -108,43 +110,42 @@ class LearnerProfileTestMixin(EventsTestMixin): """ Verifies that the correct view event was captured for the profile page. """ - self.verify_events_of_type( - requesting_username, - u"edx.user.settings.viewed", - [{ - u"user_id": int(profile_user_id), - u"page": u"profile", - u"visibility": unicode(visibility), - }] + + actual_events = self.wait_for_events( + event_filter={'event_type': 'edx.user.settings.viewed'}, number_of_matches=1) + self.assert_events_match( + [ + { + 'username': requesting_username, + 'event': { + 'user_id': int(profile_user_id), + 'page': 'profile', + 'visibility': unicode(visibility) + } + } + ], + actual_events ) - def assert_event_emitted_num_times(self, profile_user_id, setting, num_times): - """ - Verify a particular user settings change event was emitted a certain - number of times. - """ - # pylint disable=no-member - super(LearnerProfileTestMixin, self).assert_event_emitted_num_times( - self.USER_SETTINGS_CHANGED_EVENT_NAME, self.start_time, profile_user_id, num_times, setting=setting - ) + @contextmanager + def verify_pref_change_event_during(self, username, user_id, setting, **kwargs): + """Assert that a single setting changed event is emitted for the user_api_userpreference table.""" + expected_event = { + 'username': username, + 'event': { + 'setting': setting, + 'user_id': int(user_id), + 'table': 'user_api_userpreference', + 'truncated': [] + } + } + expected_event['event'].update(kwargs) - def verify_user_preference_changed_event(self, username, user_id, setting, old_value=None, new_value=None): - """ - Verifies that the correct user preference changed event was recorded. - """ - self.verify_events_of_type( - username, - self.USER_SETTINGS_CHANGED_EVENT_NAME, - [{ - u"user_id": long(user_id), - u"table": u"user_api_userpreference", - u"setting": unicode(setting), - u"old": old_value, - u"new": new_value, - u"truncated": [], - }], - expected_referers=["/u/{username}".format(username=username)], - ) + event_filter = { + 'event_type': self.USER_SETTINGS_CHANGED_EVENT_NAME, + } + with self.assert_events_match_during(event_filter=event_filter, expected_events=[expected_event]): + yield @attr('shard_4') @@ -195,12 +196,10 @@ class OwnLearnerProfilePageTest(LearnerProfileTestMixin, WebAppTest): """ username, user_id = self.log_in_as_unique_user() profile_page = self.visit_profile_page(username, privacy=self.PRIVACY_PRIVATE) - profile_page.privacy = self.PRIVACY_PUBLIC - self.verify_user_preference_changed_event( - username, user_id, "account_privacy", - old_value=self.PRIVACY_PRIVATE, # Note: default value was public, so we first change to private - new_value=self.PRIVACY_PUBLIC, - ) + with self.verify_pref_change_event_during( + username, user_id, 'account_privacy', old=self.PRIVACY_PRIVATE, new=self.PRIVACY_PUBLIC + ): + profile_page.privacy = self.PRIVACY_PUBLIC # Reload the page and verify that the profile is now public self.browser.refresh() @@ -220,12 +219,10 @@ class OwnLearnerProfilePageTest(LearnerProfileTestMixin, WebAppTest): """ username, user_id = self.log_in_as_unique_user() profile_page = self.visit_profile_page(username, privacy=self.PRIVACY_PUBLIC) - profile_page.privacy = self.PRIVACY_PRIVATE - self.verify_user_preference_changed_event( - username, user_id, "account_privacy", - old_value=None, # Note: no old value as the default preference is public - new_value=self.PRIVACY_PRIVATE, - ) + with self.verify_pref_change_event_during( + username, user_id, 'account_privacy', old=None, new=self.PRIVACY_PRIVATE + ): + profile_page.privacy = self.PRIVACY_PRIVATE # Reload the page and verify that the profile is now private self.browser.refresh() @@ -487,13 +484,14 @@ class OwnLearnerProfilePageTest(LearnerProfileTestMixin, WebAppTest): self.assert_default_image_has_public_access(profile_page) - profile_page.upload_file(filename='image.jpg') + with self.verify_pref_change_event_during( + username, user_id, 'profile_image_uploaded_at', table='auth_userprofile' + ): + profile_page.upload_file(filename='image.jpg') self.assertTrue(profile_page.image_upload_success) profile_page.visit() self.assertTrue(profile_page.image_upload_success) - self.assert_event_emitted_num_times(user_id, 'profile_image_uploaded_at', 1) - def test_user_can_see_error_for_exceeding_max_file_size_limit(self): """ Scenario: Upload profile image does not work for > 1MB image file. @@ -516,7 +514,13 @@ class OwnLearnerProfilePageTest(LearnerProfileTestMixin, WebAppTest): profile_page.visit() self.assertTrue(profile_page.profile_has_default_image) - self.assert_event_emitted_num_times(user_id, 'profile_image_uploaded_at', 0) + self.assert_no_matching_events_were_emitted({ + 'event_type': self.USER_SETTINGS_CHANGED_EVENT_NAME, + 'event': { + 'setting': 'profile_image_uploaded_at', + 'user_id': int(user_id), + } + }) def test_user_can_see_error_for_file_size_below_the_min_limit(self): """ @@ -540,7 +544,13 @@ class OwnLearnerProfilePageTest(LearnerProfileTestMixin, WebAppTest): profile_page.visit() self.assertTrue(profile_page.profile_has_default_image) - self.assert_event_emitted_num_times(user_id, 'profile_image_uploaded_at', 0) + self.assert_no_matching_events_were_emitted({ + 'event_type': self.USER_SETTINGS_CHANGED_EVENT_NAME, + 'event': { + 'setting': 'profile_image_uploaded_at', + 'user_id': int(user_id), + } + }) def test_user_can_see_error_for_wrong_file_type(self): """ @@ -567,7 +577,13 @@ class OwnLearnerProfilePageTest(LearnerProfileTestMixin, WebAppTest): profile_page.visit() self.assertTrue(profile_page.profile_has_default_image) - self.assert_event_emitted_num_times(user_id, 'profile_image_uploaded_at', 0) + self.assert_no_matching_events_were_emitted({ + 'event_type': self.USER_SETTINGS_CHANGED_EVENT_NAME, + 'event': { + 'setting': 'profile_image_uploaded_at', + 'user_id': int(user_id), + } + }) def test_user_can_remove_profile_image(self): """ @@ -586,15 +602,21 @@ class OwnLearnerProfilePageTest(LearnerProfileTestMixin, WebAppTest): self.assert_default_image_has_public_access(profile_page) - profile_page.upload_file(filename='image.jpg') + with self.verify_pref_change_event_during( + username, user_id, 'profile_image_uploaded_at', table='auth_userprofile' + ): + profile_page.upload_file(filename='image.jpg') self.assertTrue(profile_page.image_upload_success) - self.assertTrue(profile_page.remove_profile_image()) + + with self.verify_pref_change_event_during( + username, user_id, 'profile_image_uploaded_at', table='auth_userprofile' + ): + self.assertTrue(profile_page.remove_profile_image()) + self.assertTrue(profile_page.profile_has_default_image) profile_page.visit() self.assertTrue(profile_page.profile_has_default_image) - self.assert_event_emitted_num_times(user_id, 'profile_image_uploaded_at', 2) - def test_user_cannot_remove_default_image(self): """ Scenario: Remove profile image does not works for default images. @@ -623,10 +645,17 @@ class OwnLearnerProfilePageTest(LearnerProfileTestMixin, WebAppTest): username, user_id = self.log_in_as_unique_user() profile_page = self.visit_profile_page(username, privacy=self.PRIVACY_PUBLIC) self.assert_default_image_has_public_access(profile_page) - profile_page.upload_file(filename='image.jpg') + + with self.verify_pref_change_event_during( + username, user_id, 'profile_image_uploaded_at', table='auth_userprofile' + ): + profile_page.upload_file(filename='image.jpg') self.assertTrue(profile_page.image_upload_success) - profile_page.upload_file(filename='image.jpg', wait_for_upload_button=False) - self.assert_event_emitted_num_times(user_id, 'profile_image_uploaded_at', 2) + + with self.verify_pref_change_event_during( + username, user_id, 'profile_image_uploaded_at', table='auth_userprofile' + ): + profile_page.upload_file(filename='image.jpg', wait_for_upload_button=False) @attr('shard_4') diff --git a/common/test/acceptance/tests/lms/test_lms.py b/common/test/acceptance/tests/lms/test_lms.py index 3b478ad0f0..94e2829b29 100644 --- a/common/test/acceptance/tests/lms/test_lms.py +++ b/common/test/acceptance/tests/lms/test_lms.py @@ -276,22 +276,6 @@ class PayAndVerifyTest(EventsTestMixin, UniqueCourseTest): # Submit payment self.fake_payment_page.submit_payment() - # Expect enrollment activated event - self.assert_event_emitted_num_times( - "edx.course.enrollment.activated", - self.start_time, - student_id, - 1 - ) - - # Expect that one mode_changed enrollment event fired as part of the upgrade - self.assert_event_emitted_num_times( - "edx.course.enrollment.mode_changed", - self.start_time, - student_id, - 1 - ) - # Proceed to verification self.payment_and_verification_flow.immediate_verification() @@ -329,14 +313,6 @@ class PayAndVerifyTest(EventsTestMixin, UniqueCourseTest): # Submit payment self.fake_payment_page.submit_payment() - # Expect enrollment activated event - self.assert_event_emitted_num_times( - "edx.course.enrollment.activated", - self.start_time, - student_id, - 1 - ) - # Navigate to the dashboard self.dashboard_page.visit() @@ -364,24 +340,23 @@ class PayAndVerifyTest(EventsTestMixin, UniqueCourseTest): # Proceed to the fake payment page self.upgrade_page.proceed_to_payment() - # Submit payment - self.fake_payment_page.submit_payment() + def only_enrollment_events(event): + """Filter out all non-enrollment events.""" + return event['event_type'].startswith('edx.course.enrollment.') - # Expect that one mode_changed enrollment event fired as part of the upgrade - self.assert_event_emitted_num_times( - "edx.course.enrollment.mode_changed", - self.start_time, - student_id, - 1 - ) + expected_events = [ + { + 'event_type': 'edx.course.enrollment.mode_changed', + 'event': { + 'user_id': int(student_id), + 'mode': 'verified', + } + } + ] - # Expect no enrollment activated event - self.assert_event_emitted_num_times( - "edx.course.enrollment.activated", - self.start_time, - student_id, - 0 - ) + with self.assert_events_match_during(event_filter=only_enrollment_events, expected_events=expected_events): + # Submit payment + self.fake_payment_page.submit_payment() # Navigate to the dashboard self.dashboard_page.visit() diff --git a/common/test/acceptance/tests/video/test_video_events.py b/common/test/acceptance/tests/video/test_video_events.py new file mode 100644 index 0000000000..dabe17bda2 --- /dev/null +++ b/common/test/acceptance/tests/video/test_video_events.py @@ -0,0 +1,141 @@ +"""Ensure videos emit proper events""" + +import datetime +import json + +from ..helpers import EventsTestMixin +from .test_video_module import VideoBaseTest + +from openedx.core.lib.tests.assertions.events import assert_event_matches, assert_events_equal +from opaque_keys.edx.keys import UsageKey, CourseKey + + +class VideoEventsTest(EventsTestMixin, VideoBaseTest): + """ Test video player event emission """ + + def test_video_control_events(self): + """ + Scenario: Video component is rendered in the LMS in Youtube mode without HTML5 sources + Given the course has a Video component in "Youtube" mode + And I play the video + And I watch 5 seconds of it + And I pause the video + Then a "load_video" event is emitted + And a "play_video" event is emitted + And a "pause_video" event is emitted + """ + + def is_video_event(event): + """Filter out anything other than the video events of interest""" + return event['event_type'] in ('load_video', 'play_video', 'pause_video') + + captured_events = [] + with self.capture_events(is_video_event, number_of_matches=3, captured_events=captured_events): + self.navigate_to_video() + self.video.click_player_button('play') + self.video.wait_for_position('0:05') + self.video.click_player_button('pause') + + for idx, video_event in enumerate(captured_events): + self.assert_payload_contains_ids(video_event) + if idx == 0: + assert_event_matches({'event_type': 'load_video'}, video_event) + elif idx == 1: + assert_event_matches({'event_type': 'play_video'}, video_event) + self.assert_valid_control_event_at_time(video_event, 0) + elif idx == 2: + assert_event_matches({'event_type': 'pause_video'}, video_event) + self.assert_valid_control_event_at_time(video_event, self.video.seconds) + + def assert_payload_contains_ids(self, video_event): + """ + Video events should all contain "id" and "code" attributes in their payload. + + This function asserts that those fields are present and have correct values. + """ + video_descriptors = self.course_fixture.get_nested_xblocks(category='video') + video_desc = video_descriptors[0] + video_locator = UsageKey.from_string(video_desc.locator) + + expected_event = { + 'event': { + 'id': video_locator.html_id(), + 'code': '3_yD_cEKoCk' + } + } + self.assert_events_match([expected_event], [video_event]) + + def assert_valid_control_event_at_time(self, video_event, time_in_seconds): + """ + Video control events should contain valid ID fields and a valid "currentTime" field. + + This function asserts that those fields are present and have correct values. + """ + current_time = json.loads(video_event['event'])['currentTime'] + self.assertAlmostEqual(current_time, time_in_seconds, delta=1) + + def test_strict_event_format(self): + """ + This test makes a very strong assertion about the fields present in events. The goal of it is to ensure that new + fields are not added to all events mistakenly. It should be the only existing test that is updated when new top + level fields are added to all events. + """ + + captured_events = [] + with self.capture_events(lambda e: e['event_type'] == 'load_video', captured_events=captured_events): + self.navigate_to_video() + + load_video_event = captured_events[0] + + # Validate the event payload + self.assert_payload_contains_ids(load_video_event) + + # We cannot predict the value of these fields so we make weaker assertions about them + dynamic_string_fields = ( + 'accept_language', + 'agent', + 'host', + 'ip', + 'event', + 'session' + ) + for field in dynamic_string_fields: + self.assert_field_type(load_video_event, field, basestring) + self.assertIn(field, load_video_event, '{0} not found in the root of the event'.format(field)) + del load_video_event[field] + + # A weak assertion for the timestamp as well + self.assert_field_type(load_video_event, 'time', datetime.datetime) + del load_video_event['time'] + + # Note that all unpredictable fields have been deleted from the event at this point + + course_key = CourseKey.from_string(self.course_id) + static_fields_pattern = { + 'context': { + 'course_id': unicode(course_key), + 'org_id': course_key.org, + 'path': '/event', + 'user_id': self.user_info['user_id'] + }, + 'event_source': 'browser', + 'event_type': 'load_video', + 'username': self.user_info['username'], + 'page': self.browser.current_url, + 'referer': self.browser.current_url, + 'name': 'load_video', + } + assert_events_equal(static_fields_pattern, load_video_event) + + def assert_field_type(self, event_dict, field, field_type): + """Assert that a particular `field` in the `event_dict` has a particular type""" + self.assertIn(field, event_dict, '{0} not found in the root of the event'.format(field)) + self.assertTrue( + isinstance(event_dict[field], field_type), + 'Expected "{key}" to be a "{field_type}", but it has the value "{value}" of type "{t}"'.format( + key=field, + value=event_dict[field], + t=type(event_dict[field]), + field_type=field_type, + ) + ) diff --git a/common/test/acceptance/tests/video/test_video_module.py b/common/test/acceptance/tests/video/test_video_module.py index 95902def8f..727ff7e8ba 100644 --- a/common/test/acceptance/tests/video/test_video_module.py +++ b/common/test/acceptance/tests/video/test_video_module.py @@ -48,6 +48,7 @@ class VideoBaseTest(UniqueCourseTest): self.tab_nav = TabNavPage(self.browser) self.course_nav = CourseNavPage(self.browser) self.course_info_page = CourseInfoPage(self.browser, self.course_id) + self.auth_page = AutoAuthPage(self.browser, course_id=self.course_id) self.course_fixture = CourseFixture( self.course_info['org'], self.course_info['number'], @@ -58,6 +59,7 @@ class VideoBaseTest(UniqueCourseTest): self.assets = [] self.verticals = None self.youtube_configuration = {} + self.user_info = {} # reset youtube stub server self.addCleanup(YouTubeStubConfig.reset) @@ -125,8 +127,8 @@ class VideoBaseTest(UniqueCourseTest): def _navigate_to_courseware_video(self): """ Register for the course and navigate to the video unit """ - AutoAuthPage(self.browser, course_id=self.course_id).visit() - + self.auth_page.visit() + self.user_info = self.auth_page.user_info self.course_info_page.visit() self.tab_nav.go_to_tab('Courseware') diff --git a/lms/envs/acceptance.py b/lms/envs/acceptance.py index 0f7918f1a8..776ee30627 100644 --- a/lms/envs/acceptance.py +++ b/lms/envs/acceptance.py @@ -80,7 +80,7 @@ TRACKING_BACKENDS.update({ } }) -EVENT_TRACKING_BACKENDS.update({ +EVENT_TRACKING_BACKENDS['tracking_logs']['OPTIONS']['backends'].update({ 'mongo': { 'ENGINE': 'eventtracking.backends.mongodb.MongoBackend', 'OPTIONS': { diff --git a/lms/envs/aws.py b/lms/envs/aws.py index 12346c173e..45104a17c4 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -457,7 +457,9 @@ STUDENT_FILEUPLOAD_MAX_SIZE = ENV_TOKENS.get("STUDENT_FILEUPLOAD_MAX_SIZE", STUD # Event tracking TRACKING_BACKENDS.update(AUTH_TOKENS.get("TRACKING_BACKENDS", {})) -EVENT_TRACKING_BACKENDS.update(AUTH_TOKENS.get("EVENT_TRACKING_BACKENDS", {})) +EVENT_TRACKING_BACKENDS['tracking_logs']['OPTIONS']['backends'].update(AUTH_TOKENS.get("EVENT_TRACKING_BACKENDS", {})) +EVENT_TRACKING_BACKENDS['segmentio']['OPTIONS']['processors'][0]['OPTIONS']['whitelist'].extend( + AUTH_TOKENS.get("EVENT_TRACKING_SEGMENTIO_EMIT_WHITELIST", [])) TRACKING_SEGMENTIO_WEBHOOK_SECRET = AUTH_TOKENS.get( "TRACKING_SEGMENTIO_WEBHOOK_SECRET", TRACKING_SEGMENTIO_WEBHOOK_SECRET diff --git a/lms/envs/common.py b/lms/envs/common.py index 7964ce7feb..04a6d9920b 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -587,22 +587,42 @@ TRACKING_IGNORE_URL_PATTERNS = [r'^/event', r'^/login', r'^/heartbeat', r'^/segm EVENT_TRACKING_ENABLED = True EVENT_TRACKING_BACKENDS = { - 'logger': { - 'ENGINE': 'eventtracking.backends.logger.LoggerBackend', + 'tracking_logs': { + 'ENGINE': 'eventtracking.backends.routing.RoutingBackend', 'OPTIONS': { - 'name': 'tracking', - 'max_event_size': TRACK_MAX_EVENT, + 'backends': { + 'logger': { + 'ENGINE': 'eventtracking.backends.logger.LoggerBackend', + 'OPTIONS': { + 'name': 'tracking', + 'max_event_size': TRACK_MAX_EVENT, + } + } + }, + 'processors': [ + {'ENGINE': 'track.shim.LegacyFieldMappingProcessor'}, + {'ENGINE': 'track.shim.VideoEventProcessor'} + ] + } + }, + 'segmentio': { + 'ENGINE': 'eventtracking.backends.routing.RoutingBackend', + 'OPTIONS': { + 'backends': { + 'segment': {'ENGINE': 'eventtracking.backends.segment.SegmentBackend'} + }, + 'processors': [ + { + 'ENGINE': 'eventtracking.processors.whitelist.NameWhitelistProcessor', + 'OPTIONS': { + 'whitelist': [] + } + } + ] } } } -EVENT_TRACKING_PROCESSORS = [ - { - 'ENGINE': 'track.shim.LegacyFieldMappingProcessor' - }, - { - 'ENGINE': 'track.shim.VideoEventProcessor' - } -] +EVENT_TRACKING_PROCESSORS = [] # Backwards compatibility with ENABLE_SQL_TRACKING_LOGS feature flag. # In the future, adding the backend to TRACKING_BACKENDS should be enough. diff --git a/openedx/core/lib/tests/__init__.py b/openedx/core/lib/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openedx/core/lib/tests/assertions/__init__.py b/openedx/core/lib/tests/assertions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openedx/core/lib/tests/assertions/events.py b/openedx/core/lib/tests/assertions/events.py new file mode 100644 index 0000000000..e638de2a30 --- /dev/null +++ b/openedx/core/lib/tests/assertions/events.py @@ -0,0 +1,251 @@ +"""Assertions related to event validation""" + +import json +import pprint + + +def assert_event_matches(expected, actual, tolerate=None): + """ + Compare two event dictionaries. + + Fail if any discrepancies exist, and output the list of all discrepancies. The intent is to produce clearer + error messages than "{ some massive dict } != { some other massive dict }", instead enumerating the keys that + differ. Produces period separated "paths" to keys in the output, so "context.foo" refers to the following + structure: + + { + 'context': { + 'foo': 'bar' # this key, value pair + } + } + + The other key difference between this comparison and `assertEquals` is that it supports differing levels of + tolerance for discrepancies. We don't want to litter our tests full of exact match tests because then anytime we + add a field to all events, we have to go update every single test that has a hardcoded complete event structure in + it. Instead we support making partial assertions about structure and content of the event. So if I say my expected + event looks like this: + + { + 'event_type': 'foo.bar', + 'event': { + 'user_id': 10 + } + } + + This method will raise an assertion error if the actual event either does not contain the above fields in their + exact locations in the hierarchy, or if it does contain them but has different values for them. Note that it will + *not* necessarily raise an assertion error if the actual event contains other fields that are not listed in the + expected event. For example, the following event would not raise an assertion error: + + { + 'event_type': 'foo.bar', + 'referer': 'http://example.com' + 'event': { + 'user_id': 10 + } + } + + Note that the extra "referer" field is not considered an error by default. + + The `tolerate` parameter takes a set that allows you to specify varying degrees of tolerance for some common + eventing related issues. See the `EventMatchTolerates` class for more information about the various flags that are + supported here. + + Example output if an error is found: + + Unexpected differences found in structs: + + * : not found in actual + * : != (expected != actual) + + Expected: + { } + + "" is a "." separated string indicating the key that differed. In the examples above "event.user_id" would + refer to the value of the "user_id" field contained within the dictionary referred to by the "event" field in the + root dictionary. + """ + differences = get_event_differences(expected, actual, tolerate=tolerate) + if len(differences) > 0: + debug_info = [ + '', + 'Expected:', + block_indent(expected), + 'Actual:', + block_indent(actual), + 'Tolerating:', + block_indent(EventMatchTolerates.default_if_not_defined(tolerate)), + ] + differences = ['* ' + d for d in differences] + message_lines = differences + debug_info + raise AssertionError('Unexpected differences found in structs:\n\n' + '\n'.join(message_lines)) + + +class EventMatchTolerates(object): + """ + Represents groups of flags that specify the level of tolerance for deviation between an expected event and an actual + event. + + These are common event specific deviations that we don't want to handle with special case logic throughout our + tests. + """ + + # Allow the "event" field to be a string, currently this is the case for all browser events. + STRING_PAYLOAD = 'string_payload' + + # Allow unexpected fields to exist in the top level event dictionary. + ROOT_EXTRA_FIELDS = 'root_extra_fields' + + # Allow unexpected fields to exist in the "context" dictionary. This is where new fields that appear in multiple + # events are most commonly added, so we frequently want to tolerate variation here. + CONTEXT_EXTRA_FIELDS = 'context_extra_fields' + + # Allow unexpected fields to exist in the "event" dictionary. Typically in unit tests we don't want to allow this + # type of variance since there are typically only a small number of tests for a particular event type. + PAYLOAD_EXTRA_FIELDS = 'payload_extra_fields' + + @classmethod + def default(cls): + """A reasonable set of tolerated variations.""" + # NOTE: "payload_extra_fields" is deliberately excluded from this list since we want to detect erroneously added + # fields in the payload by default. + return { + cls.STRING_PAYLOAD, + cls.ROOT_EXTRA_FIELDS, + cls.CONTEXT_EXTRA_FIELDS, + } + + @classmethod + def lenient(cls): + """Allow all known variations.""" + return cls.default() | { + cls.PAYLOAD_EXTRA_FIELDS + } + + @classmethod + def strict(cls): + """Allow no variation at all.""" + return frozenset() + + @classmethod + def default_if_not_defined(cls, tolerates=None): + """Use the provided tolerance or provide a default one if None was specified.""" + if tolerates is None: + return cls.default() + else: + return tolerates + + +def assert_events_equal(expected, actual): + """ + Strict comparison of two events. + + This asserts that every field in the real event exactly matches the expected event. + """ + assert_event_matches(expected, actual, tolerate=EventMatchTolerates.strict()) + + +def get_event_differences(expected, actual, tolerate=None): + """Given two events, gather a list of differences between them given some set of tolerated variances.""" + tolerate = EventMatchTolerates.default_if_not_defined(tolerate) + + # Some events store their payload in a JSON string instead of a dict. Comparing these strings can be problematic + # since the keys may be in different orders, so we parse the string here if we were expecting a dict. + if EventMatchTolerates.STRING_PAYLOAD in tolerate: + expected = parse_event_payload(expected) + actual = parse_event_payload(actual) + + def should_strict_compare(path): + """ + We want to be able to vary the degree of strictness we apply depending on the testing context. + + Some tests will want to assert that the entire event matches exactly, others will tolerate some variance in the + context or root fields, but not in the payload (for example). + """ + if path == [] and EventMatchTolerates.ROOT_EXTRA_FIELDS in tolerate: + return False + elif path == ['event'] and EventMatchTolerates.PAYLOAD_EXTRA_FIELDS in tolerate: + return False + elif path == ['context'] and EventMatchTolerates.CONTEXT_EXTRA_FIELDS in tolerate: + return False + else: + return True + + return compare_structs(expected, actual, should_strict_compare=should_strict_compare) + + +def block_indent(text, spaces=4): + """ + Given a multi-line string, indent every line of it by the given number of spaces. + + If `text` is not a string it is formatted using pprint.pformat. + """ + return '\n'.join([(' ' * spaces) + l for l in pprint.pformat(text).splitlines()]) + + +def parse_event_payload(event): + """ + Given an event, parse the "event" field as a JSON string. + + Note that this may simply return the same event unchanged, or return a new copy of the event with the payload + parsed. It will never modify the event in place. + """ + if 'event' in event and isinstance(event['event'], basestring): + event = event.copy() + try: + event['event'] = json.loads(event['event']) + except ValueError: + pass + return event + + +def compare_structs(expected, actual, should_strict_compare=None, path=None): + """ + Traverse two structures to ensure that the `actual` structure contains all of the elements within the `expected` + one. + + Note that this performs a "deep" comparison, descending into dictionaries, lists and ohter collections to ensure + that the structure matches the expectation. + + If a particular value is not recognized, it is simply compared using the "!=" operator. + """ + if path is None: + path = [] + differences = [] + + if isinstance(expected, dict) and isinstance(actual, dict): + expected_keys = frozenset(expected.keys()) + actual_keys = frozenset(actual.keys()) + + for key in expected_keys - actual_keys: + differences.append('{0}: not found in actual'.format(_path_to_string(path + [key]))) + + if should_strict_compare is not None and should_strict_compare(path): + for key in actual_keys - expected_keys: + differences.append('{0}: only defined in actual'.format(_path_to_string(path + [key]))) + + for key in expected_keys & actual_keys: + child_differences = compare_structs(expected[key], actual[key], should_strict_compare, path + [key]) + differences.extend(child_differences) + + elif expected != actual: + differences.append('{path}: {a} != {b} (expected != actual)'.format( + path=_path_to_string(path), + a=repr(expected), + b=repr(actual) + )) + + return differences + + +def is_matching_event(expected_event, actual_event, tolerate=None): + """Return True iff the `actual_event` matches the `expected_event` given the tolerances.""" + return len(get_event_differences(expected_event, actual_event, tolerate=tolerate)) == 0 + + +def _path_to_string(path): + """Convert a list of path elements into a single path string.""" + return '.'.join(path) diff --git a/requirements/edx/github.txt b/requirements/edx/github.txt index 4e8ea43cf2..32abc72414 100644 --- a/requirements/edx/github.txt +++ b/requirements/edx/github.txt @@ -30,7 +30,7 @@ git+https://github.com/pmitros/pyfs.git@96e1922348bfe6d99201b9512a9ed946c87b7e0b -e git+https://github.com/edx/XBlock.git@1934a2978cdd3e2414486c74b76e3040ff1fb138#egg=XBlock -e git+https://github.com/edx/codejail.git@6b17c33a89bef0ac510926b1d7fea2748b73aadd#egg=codejail -e git+https://github.com/edx/js-test-tool.git@v0.1.6#egg=js_test_tool --e git+https://github.com/edx/event-tracking.git@0.1.0#egg=event-tracking +-e git+https://github.com/edx/event-tracking.git@0.2.0#egg=event-tracking -e git+https://github.com/edx/bok-choy.git@82d2f4b72e807b10d112179c0a4abd810a001b82#egg=bok_choy -e git+https://github.com/edx-solutions/django-splash.git@7579d052afcf474ece1239153cffe1c89935bc4f#egg=django-splash -e git+https://github.com/edx/acid-block.git@e46f9cda8a03e121a00c7e347084d142d22ebfb7#egg=acid-xblock