Make user_track use eventtracking

common/test/acceptance/fixtures/discussion.py

@@ -28,6 +28,13 @@ class ContentFactory(factory.Factory):
     closed = False
     votes = {"up_count": 0}
 
+    @classmethod
+    def _adjust_kwargs(cls, **kwargs):
+        # The discussion code assumes that user_id is a string. This ensures that it always will be.
+        if 'user_id' in kwargs:
+            kwargs['user_id'] = str(kwargs['user_id'])
+        return kwargs
+
 
 class Thread(ContentFactory):
     thread_type = "discussion"

common/test/acceptance/pages/lms/auto_auth.py

@@ -4,7 +4,7 @@ Auto-auth page (used to automatically log in during testing).
 
 import re
 import urllib
-from bok_choy.page_object import PageObject
+from bok_choy.page_object import PageObject, unguarded
 from . import AUTH_BASE_URL
 
 
@@ -15,6 +15,8 @@ class AutoAuthPage(PageObject):
     this url will create a user and log them in.
     """
 
+    CONTENT_REGEX = r'.+? user (?P<username>\S+) \((?P<email>.+?)\) with password \S+ and user_id (?P<user_id>\d+)$'
+
     def __init__(self, browser, username=None, email=None, password=None, staff=None, course_id=None, roles=None):
         """
         Auto-auth is an end-point for HTTP GET requests.
@@ -30,6 +32,9 @@ class AutoAuthPage(PageObject):
         """
         super(AutoAuthPage, self).__init__(browser)
 
+        # This will eventually hold the details about the user account
+        self._user_info = None
+
         # Create query string parameters if provided
         self._params = {}
 
@@ -65,14 +70,31 @@ class AutoAuthPage(PageObject):
         return url
 
     def is_browser_on_page(self):
+        return True if self.get_user_info() is not None else False
+
+    @unguarded
+    def get_user_info(self):
+        """Parse the auto auth page body to extract relevant details about the user that was logged in."""
         message = self.q(css='BODY').text[0]
-        match = re.search(r'Logged in user ([^$]+) with password ([^$]+) and user_id ([^$]+)$', message)
-        return True if match else False
+        match = re.match(self.CONTENT_REGEX, message)
+        if not match:
+            return None
+        else:
+            user_info = match.groupdict()
+            user_info['user_id'] = int(user_info['user_id'])
+            return user_info
+
+    @property
+    def user_info(self):
+        """A dictionary containing details about the user account."""
+        if self._user_info is None:
+            user_info = self.get_user_info()
+            if user_info is not None:
+                self._user_info = self.get_user_info()
+        return self._user_info
 
     def get_user_id(self):
         """
         Finds and returns the user_id
         """
-        message = self.q(css='BODY').text[0].strip()
-        match = re.search(r' user_id ([^$]+)$', message)
-        return match.groups()[0] if match else None
+        return self.user_info['user_id']

common/test/acceptance/tests/helpers.py

@@ -1,18 +1,24 @@
 """
 Test helper functions and base classes.
 """
+import inspect
 import json
 import unittest
 import functools
+import operator
+import pprint
 import requests
 import os
+import urlparse
+from contextlib import contextmanager
 from datetime import datetime
 from path import path
 from bok_choy.javascript import js_defined
 from bok_choy.web_app_test import WebAppTest
-from bok_choy.promise import EmptyPromise
+from bok_choy.promise import EmptyPromise, Promise
 from opaque_keys.edx.locator import CourseLocator
-from pymongo import MongoClient
+from pymongo import MongoClient, ASCENDING
+from openedx.core.lib.tests.assertions.events import assert_event_matches, is_matching_event, EventMatchTolerates
 from xmodule.partitions.partitions import UserPartition
 from xmodule.partitions.tests.test_partitions import MockUserPartitionScheme
 from selenium.webdriver.support.select import Select
@@ -20,6 +26,12 @@ from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 
 
+from ..pages.common import BASE_URL
+
+
+MAX_EVENTS_IN_FAILURE_OUTPUT = 20
+
+
 def skip_if_browser(browser):
     """
     Method decorator that skips a test if browser is `browser`
@@ -279,81 +291,257 @@ class EventsTestMixin(object):
         self.event_collection = MongoClient()["test"]["events"]
         self.reset_event_tracking()
 
-    def assert_event_emitted_num_times(self, event_name, event_time, event_user_id, num_times_emitted, **kwargs):
-        """
-        Tests the number of times a particular event was emitted.
-
-        Extra kwargs get passed to the mongo query in the form: "event.<key>: value".
-
-        :param event_name: Expected event name (e.g., "edx.course.enrollment.activated")
-        :param event_time: Latest expected time, after which the event would fire (e.g., the beginning of the test case)
-        :param event_user_id: user_id expected in the event
-        :param num_times_emitted: number of times the event is expected to appear since the event_time
-        """
-        find_kwargs = {
-            "name": event_name,
-            "time": {"$gt": event_time},
-            "event.user_id": int(event_user_id),
-        }
-        find_kwargs.update({"event.{}".format(key): value for key, value in kwargs.items()})
-        matching_events = self.event_collection.find(find_kwargs)
-        self.assertEqual(matching_events.count(), num_times_emitted, '\n'.join(str(event) for event in matching_events))
-
     def reset_event_tracking(self):
-        """
-        Resets all event tracking so that previously captured events are removed.
-        """
+        """Drop any events that have been collected thus far and start collecting again from scratch."""
         self.event_collection.drop()
         self.start_time = datetime.now()
 
-    def get_matching_events(self, username, event_type):
+    @contextmanager
+    def capture_events(self, event_filter=None, number_of_matches=1, captured_events=None):
         """
-        Returns a cursor for the matching browser events related emitted for the specified username.
-        """
-        return self.event_collection.find({
-            "username": username,
-            "event_type": event_type,
-            "time": {"$gt": self.start_time},
-        })
+        Context manager that captures all events emitted while executing a particular block.
 
-    def verify_events_of_type(self, username, event_type, expected_events, expected_referers=None):
-        """Verify that the expected events of a given type were logged.
-        Args:
-            username (str): The name of the user for which events will be tested.
-            event_type (str): The type of event to be verified.
-            expected_events (list): A list of dicts representing the events that should
-                have been fired.
-            expected_referers (list): A list of strings representing the referers for each event
-                that should been fired (optional). If present, the actual referers compared
-                with this list, checking that the expected_referers are the suffixes of
-                actual_referers. For example, if one event is expected, specifying ["/account/settings"]
-                will verify that the referer for the single event ends with "/account/settings".
+        All captured events are stored in the list referenced by `captured_events`. Note that this list is appended to
+        *in place*. The events will be appended to the list in the order they are emitted.
+
+        The `event_filter` is expected to be a callable that allows you to filter the event stream and select particular
+        events of interest. A dictionary `event_filter` is also supported, which simply indicates that the event should
+        match that provided expectation.
+
+        `number_of_matches` tells this context manager when enough events have been found and it can move on. The
+        context manager will not exit until this many events have passed the filter. If not enough events are found
+        before a timeout expires, then this will raise a `BrokenPromise` error. Note that this simply states that
+        *at least* this many events have been emitted, so `number_of_matches` is simply a lower bound for the size of
+        `captured_events`.
         """
-        EmptyPromise(
-            lambda: self.get_matching_events(username, event_type).count() >= len(expected_events),
-            "Waiting for the minimum number of events of type {type} to have been recorded".format(type=event_type)
+        start_time = datetime.utcnow()
+
+        yield
+
+        events = self.wait_for_events(
+            start_time=start_time, event_filter=event_filter, number_of_matches=number_of_matches)
+
+        if captured_events is not None and hasattr(captured_events, 'append') and callable(captured_events.append):
+            for event in events:
+                captured_events.append(event)
+
+    @contextmanager
+    def assert_events_match_during(self, event_filter=None, expected_events=None):
+        """
+        Context manager that ensures that events matching the `event_filter` and `expected_events` are emitted.
+
+        This context manager will filter out the event stream using the `event_filter` and wait for
+        `len(expected_events)` to match the filter.
+
+        It will then compare the events in order with their counterpart in `expected_events` to ensure they match the
+        more detailed assertion.
+
+        Typically `event_filter` will be an `event_type` filter and the `expected_events` list will contain more
+        detailed assertions.
+        """
+        captured_events = []
+        with self.capture_events(event_filter, len(expected_events), captured_events):
+            yield
+
+        self.assert_events_match(expected_events, captured_events)
+
+    def wait_for_events(self, start_time=None, event_filter=None, number_of_matches=1, timeout=None):
+        """
+        Wait for `number_of_matches` events to pass the `event_filter`.
+
+        By default, this will look at all events that have been emitted since the beginning of the setup of this mixin.
+        A custom `start_time` can be specified which will limit the events searched to only those emitted after that
+        time.
+
+        The `event_filter` is expected to be a callable that allows you to filter the event stream and select particular
+        events of interest. A dictionary `event_filter` is also supported, which simply indicates that the event should
+        match that provided expectation.
+
+        `number_of_matches` lets us know when enough events have been found and it can move on. The function will not
+        return until this many events have passed the filter. If not enough events are found before a timeout expires,
+        then this will raise a `BrokenPromise` error. Note that this simply states that *at least* this many events have
+        been emitted, so `number_of_matches` is simply a lower bound for the size of `captured_events`.
+
+        Specifying a custom `timeout` can allow you to extend the default 30 second timeout if necessary.
+        """
+        if start_time is None:
+            start_time = self.start_time
+
+        if timeout is None:
+            timeout = 30
+
+        def check_for_matching_events():
+            """Gather any events that have been emitted since `start_time`"""
+            return self.matching_events_were_emitted(
+                start_time=start_time,
+                event_filter=event_filter,
+                number_of_matches=number_of_matches
+            )
+
+        return Promise(
+            check_for_matching_events,
+            # This is a bit of a hack, Promise calls str(description), so I set the description to an object with a
+            # custom __str__ and have it do some intelligent stuff to generate a helpful error message.
+            CollectedEventsDescription(
+                'Waiting for {number_of_matches} events to match the filter:\n{event_filter}'.format(
+                    number_of_matches=number_of_matches,
+                    event_filter=self.event_filter_to_descriptive_string(event_filter),
+                ),
+                functools.partial(self.get_matching_events_from_time, start_time=start_time, event_filter={})
+            ),
+            timeout=timeout
         ).fulfill()
 
-        # Verify that the correct events were fired
-        cursor = self.get_matching_events(username, event_type)
-        actual_events = []
-        actual_referers = []
-        for __ in range(0, cursor.count()):
-            emitted_data = cursor.next()
-            event = emitted_data["event"]
-            if emitted_data["event_source"] == "browser":
-                event = json.loads(event)
-            actual_events.append(event)
-            actual_referers.append(emitted_data["referer"])
-        self.assertEqual(expected_events, actual_events)
-        if expected_referers is not None:
-            self.assertEqual(len(expected_referers), len(actual_referers), "Number of expected referers is incorrect")
-            for index, actual_referer in enumerate(actual_referers):
-                self.assertTrue(
-                    actual_referer.endswith(expected_referers[index]),
-                    "Refer '{0}' does not have correct suffix, '{1}'.".format(actual_referer, expected_referers[index])
+    def matching_events_were_emitted(self, start_time=None, event_filter=None, number_of_matches=1):
+        """Return True if enough events have been emitted that pass the `event_filter` since `start_time`."""
+        matching_events = self.get_matching_events_from_time(start_time=start_time, event_filter=event_filter)
+        return len(matching_events) >= number_of_matches, matching_events
+
+    def get_matching_events_from_time(self, start_time=None, event_filter=None):
+        """
+        Return a list of events that pass the `event_filter` and were emitted after `start_time`.
+
+        This function is used internally by most of the other assertions and convenience methods in this class.
+
+        The `event_filter` is expected to be a callable that allows you to filter the event stream and select particular
+        events of interest. A dictionary `event_filter` is also supported, which simply indicates that the event should
+        match that provided expectation.
+        """
+        if start_time is None:
+            start_time = self.start_time
+
+        if isinstance(event_filter, dict):
+            event_filter = functools.partial(is_matching_event, event_filter)
+        elif not callable(event_filter):
+            raise ValueError(
+                'event_filter must either be a dict or a callable function with as single "event" parameter that '
+                'returns a boolean value.'
+            )
+
+        matching_events = []
+        cursor = self.event_collection.find(
+            {
+                "time": {
+                    "$gte": start_time
+                }
+            }
+        ).sort("time", ASCENDING)
+        for event in cursor:
+            matches = False
+            try:
+                # Mongo automatically assigns an _id to all events inserted into it. We strip it out here, since
+                # we don't care about it.
+                del event['_id']
+                if event_filter is not None:
+                    # Typically we will be grabbing all events of a particular type, however, you can use arbitrary
+                    # logic to identify the events that are of interest.
+                    matches = event_filter(event)
+            except AssertionError:
+                # allow the filters to use "assert" to filter out events
+                continue
+            else:
+                if matches is None or matches:
+                    matching_events.append(event)
+        return matching_events
+
+    def assert_matching_events_were_emitted(self, start_time=None, event_filter=None, number_of_matches=1):
+        """Assert that at least `number_of_matches` events have passed the filter since `start_time`."""
+        description = CollectedEventsDescription(
+            'Not enough events match the filter:\n' + self.event_filter_to_descriptive_string(event_filter),
+            functools.partial(self.get_matching_events_from_time, start_time=start_time, event_filter={})
+        )
+
+        self.assertTrue(
+            self.matching_events_were_emitted(
+                start_time=start_time, event_filter=event_filter, number_of_matches=number_of_matches
+            ),
+            description
+        )
+
+    def assert_no_matching_events_were_emitted(self, event_filter, start_time=None):
+        """Assert that no events have passed the filter since `start_time`."""
+        matching_events = self.get_matching_events_from_time(start_time=start_time, event_filter=event_filter)
+
+        description = CollectedEventsDescription(
+            'Events unexpected matched the filter:\n' + self.event_filter_to_descriptive_string(event_filter),
+            lambda: matching_events
+        )
+
+        self.assertEquals(len(matching_events), 0, description)
+
+    def assert_events_match(self, expected_events, actual_events):
+        """
+        Assert that each item in the expected events sequence matches its counterpart at the same index in the actual
+        events sequence.
+        """
+        for expected_event, actual_event in zip(expected_events, actual_events):
+            assert_event_matches(
+                expected_event,
+                actual_event,
+                tolerate=EventMatchTolerates.lenient()
+            )
+
+    def relative_path_to_absolute_uri(self, relative_path):
+        """Return an aboslute URI given a relative path taking into account the test context."""
+        return urlparse.urljoin(BASE_URL, relative_path)
+
+    def event_filter_to_descriptive_string(self, event_filter):
+        """Find the source code of the callable or pretty-print the dictionary"""
+        message = ''
+        if callable(event_filter):
+            file_name = '(unknown)'
+            try:
+                file_name = inspect.getsourcefile(event_filter)
+            except TypeError:
+                pass
+
+            try:
+                list_of_source_lines, line_no = inspect.getsourcelines(event_filter)
+            except IOError:
+                pass
+            else:
+                message = '{file_name}:{line_no}\n{hr}\n{event_filter}\n{hr}'.format(
+                    event_filter=''.join(list_of_source_lines).rstrip(),
+                    file_name=file_name,
+                    line_no=line_no,
+                    hr='-' * 20,
                 )
 
+        if not message:
+            message = '{hr}\n{event_filter}\n{hr}'.format(
+                event_filter=pprint.pformat(event_filter),
+                hr='-' * 20,
+            )
+
+        return message
+
+
+class CollectedEventsDescription(object):
+    """
+    Produce a clear error message when tests fail.
+
+    This class calls the provided `get_events_func` when converted to a string, and pretty prints the returned events.
+    """
+
+    def __init__(self, description, get_events_func):
+        self.description = description
+        self.get_events_func = get_events_func
+
+    def __str__(self):
+        message_lines = [
+            self.description,
+            'Events:'
+        ]
+        events = self.get_events_func()
+        events.sort(key=operator.itemgetter('time'), reverse=True)
+        for event in events[:MAX_EVENTS_IN_FAILURE_OUTPUT]:
+            message_lines.append(pprint.pformat(event))
+        if len(events) > MAX_EVENTS_IN_FAILURE_OUTPUT:
+            message_lines.append(
+                'Too many events to display, the remaining events were omitted. Run locally to diagnose.')
+
+        return '\n\n'.join(message_lines)
+
 
 class UniqueCourseTest(WebAppTest):
     """

common/test/acceptance/tests/lms/test_account_settings.py

@@ -33,30 +33,49 @@ class AccountSettingsTestMixin(EventsTestMixin, WebAppTest):
         user_id = auto_auth_page.get_user_id()
         return username, user_id
 
-    def assert_event_emitted_num_times(self, user_id, setting, num_times):
-        """
-        Verify a particular user settings change event was emitted a certain
-        number of times.
-        """
-        # pylint disable=no-member
-        super(AccountSettingsTestMixin, self).assert_event_emitted_num_times(
-            self.USER_SETTINGS_CHANGED_EVENT_NAME, self.start_time, user_id, num_times, setting=setting
-        )
+    def settings_changed_event_filter(self, event):
+        """Filter out any events that are not "settings changed" events."""
+        return event['event_type'] == self.USER_SETTINGS_CHANGED_EVENT_NAME
 
-    def verify_settings_changed_events(self, username, user_id, events, table=None):
-        """
-        Verify a particular set of account settings change events were fired.
-        """
-        expected_referers = [self.ACCOUNT_SETTINGS_REFERER] * len(events)
-        for event in events:
-            event[u"user_id"] = long(user_id)
-            event[u"table"] = u"auth_userprofile" if table is None else table
-            event[u"truncated"] = []
+    def expected_settings_changed_event(self, setting, old, new, table=None):
+        """A dictionary representing the expected fields in a "settings changed" event."""
+        return {
+            'username': self.username,
+            'referer': self.get_settings_page_url(),
+            'event': {
+                'user_id': self.user_id,
+                'setting': setting,
+                'old': old,
+                'new': new,
+                'truncated': [],
+                'table': table or 'auth_userprofile'
+            }
+        }
 
-        self.verify_events_of_type(
-            username, self.USER_SETTINGS_CHANGED_EVENT_NAME, events,
-            expected_referers=expected_referers
-        )
+    def settings_change_initiated_event_filter(self, event):
+        """Filter out any events that are not "settings change initiated" events."""
+        return event['event_type'] == self.CHANGE_INITIATED_EVENT_NAME
+
+    def expected_settings_change_initiated_event(self, setting, old, new, username=None, user_id=None):
+        """A dictionary representing the expected fields in a "settings change initiated" event."""
+        return {
+            'username': username or self.username,
+            'referer': self.get_settings_page_url(),
+            'event': {
+                'user_id': user_id or self.user_id,
+                'setting': setting,
+                'old': old,
+                'new': new,
+            }
+        }
+
+    def get_settings_page_url(self):
+        """The absolute URL of the account settings page given the test context."""
+        return self.relative_path_to_absolute_uri(self.ACCOUNT_SETTINGS_REFERER)
+
+    def assert_no_setting_changed_event(self):
+        """Assert no setting changed event has been emitted thus far."""
+        self.assert_no_matching_events_were_emitted({'event_type': self.USER_SETTINGS_CHANGED_EVENT_NAME})
 
 
 @attr('shard_5')
@@ -114,14 +133,20 @@ class AccountSettingsPageTest(AccountSettingsTestMixin, WebAppTest):
         And I visit my account settings page
         Then a page view analytics event should be recorded
         """
-        self.verify_events_of_type(
-            self.username,
-            u"edx.user.settings.viewed",
-            [{
-                u"user_id": long(self.user_id),
-                u"page": u"account",
-                u"visibility": None,
-            }]
+
+        actual_events = self.wait_for_events(
+            event_filter={'event_type': 'edx.user.settings.viewed'}, number_of_matches=1)
+        self.assert_events_match(
+            [
+                {
+                    'event': {
+                        'user_id': self.user_id,
+                        'page': 'account',
+                        'visibility': None
+                    }
+                }
+            ],
+            actual_events
         )
 
     def test_all_sections_and_fields_are_present(self):
@@ -237,20 +262,13 @@ class AccountSettingsPageTest(AccountSettingsTestMixin, WebAppTest):
             [u'another name', self.username],
         )
 
-        self.verify_settings_changed_events(
-            self.username, self.user_id,
+        actual_events = self.wait_for_events(event_filter=self.settings_changed_event_filter, number_of_matches=2)
+        self.assert_events_match(
             [
-                {
-                    u"setting": u"name",
-                    u"old": self.username,
-                    u"new": u"another name",
-                },
-                {
-                    u"setting": u"name",
-                    u"old": u'another name',
-                    u"new": self.username,
-                }
-            ]
+                self.expected_settings_changed_event('name', self.username, 'another name'),
+                self.expected_settings_changed_event('name', 'another name', self.username),
+            ],
+            actual_events
         )
 
     def test_email_field(self):
@@ -270,28 +288,21 @@ class AccountSettingsPageTest(AccountSettingsTestMixin, WebAppTest):
             assert_after_reload=False
         )
 
-        self.verify_events_of_type(
-            username,
-            self.CHANGE_INITIATED_EVENT_NAME,
+        actual_events = self.wait_for_events(
+            event_filter=self.settings_change_initiated_event_filter, number_of_matches=2)
+        self.assert_events_match(
             [
-                {
-                    u"user_id": long(user_id),
-                    u"setting": u"email",
-                    u"old": email,
-                    u"new": u'me@here.com'
-                },
-                {
-                    u"user_id": long(user_id),
-                    u"setting": u"email",
-                    u"old": email,  # NOTE the first email change was never confirmed, so old has not changed.
-                    u"new": u'you@there.com'
-                }
+                self.expected_settings_change_initiated_event(
+                    'email', email, 'me@here.com', username=username, user_id=user_id),
+                # NOTE the first email change was never confirmed, so old has not changed.
+                self.expected_settings_change_initiated_event(
+                    'email', email, 'you@there.com', username=username, user_id=user_id),
             ],
-            [self.ACCOUNT_SETTINGS_REFERER, self.ACCOUNT_SETTINGS_REFERER]
+            actual_events
         )
         # Email is not saved until user confirms, so no events should have been
         # emitted.
-        self.assert_event_emitted_num_times(user_id, 'email', 0)
+        self.assert_no_setting_changed_event()
 
     def test_password_field(self):
         """
@@ -304,20 +315,11 @@ class AccountSettingsPageTest(AccountSettingsTestMixin, WebAppTest):
             success_message='Click the link in the message to reset your password.',
         )
 
-        self.verify_events_of_type(
-            self.username,
-            self.CHANGE_INITIATED_EVENT_NAME,
-            [{
-                u"user_id": int(self.user_id),
-                u"setting": "password",
-                u"old": None,
-                u"new": None
-            }],
-            [self.ACCOUNT_SETTINGS_REFERER]
-        )
+        event_filter = self.expected_settings_change_initiated_event('password', None, None)
+        self.wait_for_events(event_filter=event_filter, number_of_matches=1)
         # Like email, since the user has not confirmed their password change,
         # the field has not yet changed, so no events will have been emitted.
-        self.assert_event_emitted_num_times(self.user_id, 'password', 0)
+        self.assert_no_setting_changed_event()
 
     @skip(
         'On bokchoy test servers, language changes take a few reloads to fully realize '
@@ -345,20 +347,14 @@ class AccountSettingsPageTest(AccountSettingsTestMixin, WebAppTest):
             u'',
             [u'Bachelor\'s degree', u''],
         )
-        self.verify_settings_changed_events(
-            self.username, self.user_id,
+
+        actual_events = self.wait_for_events(event_filter=self.settings_changed_event_filter, number_of_matches=2)
+        self.assert_events_match(
             [
-                {
-                    u"setting": u"level_of_education",
-                    u"old": None,
-                    u"new": u'b',
-                },
-                {
-                    u"setting": u"level_of_education",
-                    u"old": u'b',
-                    u"new": None,
-                }
-            ]
+                self.expected_settings_changed_event('level_of_education', None, 'b'),
+                self.expected_settings_changed_event('level_of_education', 'b', None),
+            ],
+            actual_events
         )
 
     def test_gender_field(self):
@@ -371,20 +367,14 @@ class AccountSettingsPageTest(AccountSettingsTestMixin, WebAppTest):
             u'',
             [u'Female', u''],
         )
-        self.verify_settings_changed_events(
-            self.username, self.user_id,
+
+        actual_events = self.wait_for_events(event_filter=self.settings_changed_event_filter, number_of_matches=2)
+        self.assert_events_match(
             [
-                {
-                    u"setting": u"gender",
-                    u"old": None,
-                    u"new": u'f',
-                },
-                {
-                    u"setting": u"gender",
-                    u"old": u'f',
-                    u"new": None,
-                }
-            ]
+                self.expected_settings_changed_event('gender', None, 'f'),
+                self.expected_settings_changed_event('gender', 'f', None),
+            ],
+            actual_events
         )
 
     def test_year_of_birth_field(self):
@@ -393,28 +383,18 @@ class AccountSettingsPageTest(AccountSettingsTestMixin, WebAppTest):
         """
         # Note that when we clear the year_of_birth here we're firing an event.
         self.assertEqual(self.account_settings_page.value_for_dropdown_field('year_of_birth', ''), '')
-        self.reset_event_tracking()
-        self._test_dropdown_field(
-            u'year_of_birth',
-            u'Year of Birth',
-            u'',
-            [u'1980', u''],
-        )
-        self.verify_settings_changed_events(
-            self.username, self.user_id,
-            [
-                {
-                    u"setting": u"year_of_birth",
-                    u"old": None,
-                    u"new": 1980L,
-                },
-                {
-                    u"setting": u"year_of_birth",
-                    u"old": 1980L,
-                    u"new": None,
-                }
-            ]
-        )
+
+        expected_events = [
+            self.expected_settings_changed_event('year_of_birth', None, 1980),
+            self.expected_settings_changed_event('year_of_birth', 1980, None),
+        ]
+        with self.assert_events_match_during(self.settings_changed_event_filter, expected_events):
+            self._test_dropdown_field(
+                u'year_of_birth',
+                u'Year of Birth',
+                u'',
+                [u'1980', u''],
+            )
 
     def test_country_field(self):
         """
@@ -438,21 +418,15 @@ class AccountSettingsPageTest(AccountSettingsTestMixin, WebAppTest):
             [u'Pushto', u''],
         )
 
-        self.verify_settings_changed_events(
-            self.username, self.user_id,
+        actual_events = self.wait_for_events(event_filter=self.settings_changed_event_filter, number_of_matches=2)
+        self.assert_events_match(
             [
-                {
-                    u"setting": u"language_proficiencies",
-                    u"old": [],
-                    u"new": [{u"code": u"ps"}],
-                },
-                {
-                    u"setting": u"language_proficiencies",
-                    u"old": [{u"code": u"ps"}],
-                    u"new": [],
-                }
+                self.expected_settings_changed_event(
+                    'language_proficiencies', [], [{'code': 'ps'}], table='student_languageproficiency'),
+                self.expected_settings_changed_event(
+                    'language_proficiencies', [{'code': 'ps'}], [], table='student_languageproficiency'),
             ],
-            table=u"student_languageproficiency"
+            actual_events
         )
 
     def test_connected_accounts(self):

common/test/acceptance/tests/lms/test_learner_profile.py

@@ -2,6 +2,8 @@
 """
 End-to-end tests for Student's Profile Page.
 """
+from contextlib import contextmanager
+
 from datetime import datetime
 from bok_choy.web_app_test import WebAppTest
 from nose.plugins.attrib import attr
@@ -108,43 +110,42 @@ class LearnerProfileTestMixin(EventsTestMixin):
         """
         Verifies that the correct view event was captured for the profile page.
         """
-        self.verify_events_of_type(
-            requesting_username,
-            u"edx.user.settings.viewed",
-            [{
-                u"user_id": int(profile_user_id),
-                u"page": u"profile",
-                u"visibility": unicode(visibility),
-            }]
+
+        actual_events = self.wait_for_events(
+            event_filter={'event_type': 'edx.user.settings.viewed'}, number_of_matches=1)
+        self.assert_events_match(
+            [
+                {
+                    'username': requesting_username,
+                    'event': {
+                        'user_id': int(profile_user_id),
+                        'page': 'profile',
+                        'visibility': unicode(visibility)
+                    }
+                }
+            ],
+            actual_events
         )
 
-    def assert_event_emitted_num_times(self, profile_user_id, setting, num_times):
-        """
-        Verify a particular user settings change event was emitted a certain
-        number of times.
-        """
-        # pylint disable=no-member
-        super(LearnerProfileTestMixin, self).assert_event_emitted_num_times(
-            self.USER_SETTINGS_CHANGED_EVENT_NAME, self.start_time, profile_user_id, num_times, setting=setting
-        )
+    @contextmanager
+    def verify_pref_change_event_during(self, username, user_id, setting, **kwargs):
+        """Assert that a single setting changed event is emitted for the user_api_userpreference table."""
+        expected_event = {
+            'username': username,
+            'event': {
+                'setting': setting,
+                'user_id': int(user_id),
+                'table': 'user_api_userpreference',
+                'truncated': []
+            }
+        }
+        expected_event['event'].update(kwargs)
 
-    def verify_user_preference_changed_event(self, username, user_id, setting, old_value=None, new_value=None):
-        """
-        Verifies that the correct user preference changed event was recorded.
-        """
-        self.verify_events_of_type(
-            username,
-            self.USER_SETTINGS_CHANGED_EVENT_NAME,
-            [{
-                u"user_id": long(user_id),
-                u"table": u"user_api_userpreference",
-                u"setting": unicode(setting),
-                u"old": old_value,
-                u"new": new_value,
-                u"truncated": [],
-            }],
-            expected_referers=["/u/{username}".format(username=username)],
-        )
+        event_filter = {
+            'event_type': self.USER_SETTINGS_CHANGED_EVENT_NAME,
+        }
+        with self.assert_events_match_during(event_filter=event_filter, expected_events=[expected_event]):
+            yield
 
 
 @attr('shard_4')
@@ -195,12 +196,10 @@ class OwnLearnerProfilePageTest(LearnerProfileTestMixin, WebAppTest):
         """
         username, user_id = self.log_in_as_unique_user()
         profile_page = self.visit_profile_page(username, privacy=self.PRIVACY_PRIVATE)
-        profile_page.privacy = self.PRIVACY_PUBLIC
-        self.verify_user_preference_changed_event(
-            username, user_id, "account_privacy",
-            old_value=self.PRIVACY_PRIVATE,    # Note: default value was public, so we first change to private
-            new_value=self.PRIVACY_PUBLIC,
-        )
+        with self.verify_pref_change_event_during(
+            username, user_id, 'account_privacy', old=self.PRIVACY_PRIVATE, new=self.PRIVACY_PUBLIC
+        ):
+            profile_page.privacy = self.PRIVACY_PUBLIC
 
         # Reload the page and verify that the profile is now public
         self.browser.refresh()
@@ -220,12 +219,10 @@ class OwnLearnerProfilePageTest(LearnerProfileTestMixin, WebAppTest):
         """
         username, user_id = self.log_in_as_unique_user()
         profile_page = self.visit_profile_page(username, privacy=self.PRIVACY_PUBLIC)
-        profile_page.privacy = self.PRIVACY_PRIVATE
-        self.verify_user_preference_changed_event(
-            username, user_id, "account_privacy",
-            old_value=None,  # Note: no old value as the default preference is public
-            new_value=self.PRIVACY_PRIVATE,
-        )
+        with self.verify_pref_change_event_during(
+            username, user_id, 'account_privacy', old=None, new=self.PRIVACY_PRIVATE
+        ):
+            profile_page.privacy = self.PRIVACY_PRIVATE
 
         # Reload the page and verify that the profile is now private
         self.browser.refresh()
@@ -487,13 +484,14 @@ class OwnLearnerProfilePageTest(LearnerProfileTestMixin, WebAppTest):
 
         self.assert_default_image_has_public_access(profile_page)
 
-        profile_page.upload_file(filename='image.jpg')
+        with self.verify_pref_change_event_during(
+            username, user_id, 'profile_image_uploaded_at', table='auth_userprofile'
+        ):
+            profile_page.upload_file(filename='image.jpg')
         self.assertTrue(profile_page.image_upload_success)
         profile_page.visit()
         self.assertTrue(profile_page.image_upload_success)
 
-        self.assert_event_emitted_num_times(user_id, 'profile_image_uploaded_at', 1)
-
     def test_user_can_see_error_for_exceeding_max_file_size_limit(self):
         """
         Scenario: Upload profile image does not work for > 1MB image file.
@@ -516,7 +514,13 @@ class OwnLearnerProfilePageTest(LearnerProfileTestMixin, WebAppTest):
         profile_page.visit()
         self.assertTrue(profile_page.profile_has_default_image)
 
-        self.assert_event_emitted_num_times(user_id, 'profile_image_uploaded_at', 0)
+        self.assert_no_matching_events_were_emitted({
+            'event_type': self.USER_SETTINGS_CHANGED_EVENT_NAME,
+            'event': {
+                'setting': 'profile_image_uploaded_at',
+                'user_id': int(user_id),
+            }
+        })
 
     def test_user_can_see_error_for_file_size_below_the_min_limit(self):
         """
@@ -540,7 +544,13 @@ class OwnLearnerProfilePageTest(LearnerProfileTestMixin, WebAppTest):
         profile_page.visit()
         self.assertTrue(profile_page.profile_has_default_image)
 
-        self.assert_event_emitted_num_times(user_id, 'profile_image_uploaded_at', 0)
+        self.assert_no_matching_events_were_emitted({
+            'event_type': self.USER_SETTINGS_CHANGED_EVENT_NAME,
+            'event': {
+                'setting': 'profile_image_uploaded_at',
+                'user_id': int(user_id),
+            }
+        })
 
     def test_user_can_see_error_for_wrong_file_type(self):
         """
@@ -567,7 +577,13 @@ class OwnLearnerProfilePageTest(LearnerProfileTestMixin, WebAppTest):
         profile_page.visit()
         self.assertTrue(profile_page.profile_has_default_image)
 
-        self.assert_event_emitted_num_times(user_id, 'profile_image_uploaded_at', 0)
+        self.assert_no_matching_events_were_emitted({
+            'event_type': self.USER_SETTINGS_CHANGED_EVENT_NAME,
+            'event': {
+                'setting': 'profile_image_uploaded_at',
+                'user_id': int(user_id),
+            }
+        })
 
     def test_user_can_remove_profile_image(self):
         """
@@ -586,15 +602,21 @@ class OwnLearnerProfilePageTest(LearnerProfileTestMixin, WebAppTest):
 
         self.assert_default_image_has_public_access(profile_page)
 
-        profile_page.upload_file(filename='image.jpg')
+        with self.verify_pref_change_event_during(
+            username, user_id, 'profile_image_uploaded_at', table='auth_userprofile'
+        ):
+            profile_page.upload_file(filename='image.jpg')
         self.assertTrue(profile_page.image_upload_success)
-        self.assertTrue(profile_page.remove_profile_image())
+
+        with self.verify_pref_change_event_during(
+            username, user_id, 'profile_image_uploaded_at', table='auth_userprofile'
+        ):
+            self.assertTrue(profile_page.remove_profile_image())
+
         self.assertTrue(profile_page.profile_has_default_image)
         profile_page.visit()
         self.assertTrue(profile_page.profile_has_default_image)
 
-        self.assert_event_emitted_num_times(user_id, 'profile_image_uploaded_at', 2)
-
     def test_user_cannot_remove_default_image(self):
         """
         Scenario: Remove profile image does not works for default images.
@@ -623,10 +645,17 @@ class OwnLearnerProfilePageTest(LearnerProfileTestMixin, WebAppTest):
         username, user_id = self.log_in_as_unique_user()
         profile_page = self.visit_profile_page(username, privacy=self.PRIVACY_PUBLIC)
         self.assert_default_image_has_public_access(profile_page)
-        profile_page.upload_file(filename='image.jpg')
+
+        with self.verify_pref_change_event_during(
+            username, user_id, 'profile_image_uploaded_at', table='auth_userprofile'
+        ):
+            profile_page.upload_file(filename='image.jpg')
         self.assertTrue(profile_page.image_upload_success)
-        profile_page.upload_file(filename='image.jpg', wait_for_upload_button=False)
-        self.assert_event_emitted_num_times(user_id, 'profile_image_uploaded_at', 2)
+
+        with self.verify_pref_change_event_during(
+            username, user_id, 'profile_image_uploaded_at', table='auth_userprofile'
+        ):
+            profile_page.upload_file(filename='image.jpg', wait_for_upload_button=False)
 
 
 @attr('shard_4')

common/test/acceptance/tests/lms/test_lms.py

@@ -276,22 +276,6 @@ class PayAndVerifyTest(EventsTestMixin, UniqueCourseTest):
         # Submit payment
         self.fake_payment_page.submit_payment()
 
-        # Expect enrollment activated event
-        self.assert_event_emitted_num_times(
-            "edx.course.enrollment.activated",
-            self.start_time,
-            student_id,
-            1
-        )
-
-        # Expect that one mode_changed enrollment event fired as part of the upgrade
-        self.assert_event_emitted_num_times(
-            "edx.course.enrollment.mode_changed",
-            self.start_time,
-            student_id,
-            1
-        )
-
         # Proceed to verification
         self.payment_and_verification_flow.immediate_verification()
 
@@ -329,14 +313,6 @@ class PayAndVerifyTest(EventsTestMixin, UniqueCourseTest):
         # Submit payment
         self.fake_payment_page.submit_payment()
 
-        # Expect enrollment activated event
-        self.assert_event_emitted_num_times(
-            "edx.course.enrollment.activated",
-            self.start_time,
-            student_id,
-            1
-        )
-
         # Navigate to the dashboard
         self.dashboard_page.visit()
 
@@ -364,24 +340,23 @@ class PayAndVerifyTest(EventsTestMixin, UniqueCourseTest):
         # Proceed to the fake payment page
         self.upgrade_page.proceed_to_payment()
 
-        # Submit payment
-        self.fake_payment_page.submit_payment()
+        def only_enrollment_events(event):
+            """Filter out all non-enrollment events."""
+            return event['event_type'].startswith('edx.course.enrollment.')
 
-        # Expect that one mode_changed enrollment event fired as part of the upgrade
-        self.assert_event_emitted_num_times(
-            "edx.course.enrollment.mode_changed",
-            self.start_time,
-            student_id,
-            1
-        )
+        expected_events = [
+            {
+                'event_type': 'edx.course.enrollment.mode_changed',
+                'event': {
+                    'user_id': int(student_id),
+                    'mode': 'verified',
+                }
+            }
+        ]
 
-        # Expect no enrollment activated event
-        self.assert_event_emitted_num_times(
-            "edx.course.enrollment.activated",
-            self.start_time,
-            student_id,
-            0
-        )
+        with self.assert_events_match_during(event_filter=only_enrollment_events, expected_events=expected_events):
+            # Submit payment
+            self.fake_payment_page.submit_payment()
 
         # Navigate to the dashboard
         self.dashboard_page.visit()

common/test/acceptance/tests/video/test_video_events.py

@@ -0,0 +1,141 @@
+"""Ensure videos emit proper events"""
+
+import datetime
+import json
+
+from ..helpers import EventsTestMixin
+from .test_video_module import VideoBaseTest
+
+from openedx.core.lib.tests.assertions.events import assert_event_matches, assert_events_equal
+from opaque_keys.edx.keys import UsageKey, CourseKey
+
+
+class VideoEventsTest(EventsTestMixin, VideoBaseTest):
+    """ Test video player event emission """
+
+    def test_video_control_events(self):
+        """
+        Scenario: Video component is rendered in the LMS in Youtube mode without HTML5 sources
+        Given the course has a Video component in "Youtube" mode
+        And I play the video
+        And I watch 5 seconds of it
+        And I pause the video
+        Then a "load_video" event is emitted
+        And a "play_video" event is emitted
+        And a "pause_video" event is emitted
+        """
+
+        def is_video_event(event):
+            """Filter out anything other than the video events of interest"""
+            return event['event_type'] in ('load_video', 'play_video', 'pause_video')
+
+        captured_events = []
+        with self.capture_events(is_video_event, number_of_matches=3, captured_events=captured_events):
+            self.navigate_to_video()
+            self.video.click_player_button('play')
+            self.video.wait_for_position('0:05')
+            self.video.click_player_button('pause')
+
+        for idx, video_event in enumerate(captured_events):
+            self.assert_payload_contains_ids(video_event)
+            if idx == 0:
+                assert_event_matches({'event_type': 'load_video'}, video_event)
+            elif idx == 1:
+                assert_event_matches({'event_type': 'play_video'}, video_event)
+                self.assert_valid_control_event_at_time(video_event, 0)
+            elif idx == 2:
+                assert_event_matches({'event_type': 'pause_video'}, video_event)
+                self.assert_valid_control_event_at_time(video_event, self.video.seconds)
+
+    def assert_payload_contains_ids(self, video_event):
+        """
+        Video events should all contain "id" and "code" attributes in their payload.
+
+        This function asserts that those fields are present and have correct values.
+        """
+        video_descriptors = self.course_fixture.get_nested_xblocks(category='video')
+        video_desc = video_descriptors[0]
+        video_locator = UsageKey.from_string(video_desc.locator)
+
+        expected_event = {
+            'event': {
+                'id': video_locator.html_id(),
+                'code': '3_yD_cEKoCk'
+            }
+        }
+        self.assert_events_match([expected_event], [video_event])
+
+    def assert_valid_control_event_at_time(self, video_event, time_in_seconds):
+        """
+        Video control events should contain valid ID fields and a valid "currentTime" field.
+
+        This function asserts that those fields are present and have correct values.
+        """
+        current_time = json.loads(video_event['event'])['currentTime']
+        self.assertAlmostEqual(current_time, time_in_seconds, delta=1)
+
+    def test_strict_event_format(self):
+        """
+        This test makes a very strong assertion about the fields present in events. The goal of it is to ensure that new
+        fields are not added to all events mistakenly. It should be the only existing test that is updated when new top
+        level fields are added to all events.
+        """
+
+        captured_events = []
+        with self.capture_events(lambda e: e['event_type'] == 'load_video', captured_events=captured_events):
+            self.navigate_to_video()
+
+        load_video_event = captured_events[0]
+
+        # Validate the event payload
+        self.assert_payload_contains_ids(load_video_event)
+
+        # We cannot predict the value of these fields so we make weaker assertions about them
+        dynamic_string_fields = (
+            'accept_language',
+            'agent',
+            'host',
+            'ip',
+            'event',
+            'session'
+        )
+        for field in dynamic_string_fields:
+            self.assert_field_type(load_video_event, field, basestring)
+            self.assertIn(field, load_video_event, '{0} not found in the root of the event'.format(field))
+            del load_video_event[field]
+
+        # A weak assertion for the timestamp as well
+        self.assert_field_type(load_video_event, 'time', datetime.datetime)
+        del load_video_event['time']
+
+        # Note that all unpredictable fields have been deleted from the event at this point
+
+        course_key = CourseKey.from_string(self.course_id)
+        static_fields_pattern = {
+            'context': {
+                'course_id': unicode(course_key),
+                'org_id': course_key.org,
+                'path': '/event',
+                'user_id': self.user_info['user_id']
+            },
+            'event_source': 'browser',
+            'event_type': 'load_video',
+            'username': self.user_info['username'],
+            'page': self.browser.current_url,
+            'referer': self.browser.current_url,
+            'name': 'load_video',
+        }
+        assert_events_equal(static_fields_pattern, load_video_event)
+
+    def assert_field_type(self, event_dict, field, field_type):
+        """Assert that a particular `field` in the `event_dict` has a particular type"""
+        self.assertIn(field, event_dict, '{0} not found in the root of the event'.format(field))
+        self.assertTrue(
+            isinstance(event_dict[field], field_type),
+            'Expected "{key}" to be a "{field_type}", but it has the value "{value}" of type "{t}"'.format(
+                key=field,
+                value=event_dict[field],
+                t=type(event_dict[field]),
+                field_type=field_type,
+            )
+        )

common/test/acceptance/tests/video/test_video_module.py

@@ -48,6 +48,7 @@ class VideoBaseTest(UniqueCourseTest):
         self.tab_nav = TabNavPage(self.browser)
         self.course_nav = CourseNavPage(self.browser)
         self.course_info_page = CourseInfoPage(self.browser, self.course_id)
+        self.auth_page = AutoAuthPage(self.browser, course_id=self.course_id)
 
         self.course_fixture = CourseFixture(
             self.course_info['org'], self.course_info['number'],
@@ -58,6 +59,7 @@ class VideoBaseTest(UniqueCourseTest):
         self.assets = []
         self.verticals = None
         self.youtube_configuration = {}
+        self.user_info = {}
 
         # reset youtube stub server
         self.addCleanup(YouTubeStubConfig.reset)
@@ -125,8 +127,8 @@ class VideoBaseTest(UniqueCourseTest):
 
     def _navigate_to_courseware_video(self):
         """ Register for the course and navigate to the video unit """
-        AutoAuthPage(self.browser, course_id=self.course_id).visit()
-
+        self.auth_page.visit()
+        self.user_info = self.auth_page.user_info
         self.course_info_page.visit()
         self.tab_nav.go_to_tab('Courseware')