Implements keyword sub feature for bulk emails

This commit pulls in changes from #4487 that implements keyword
substitution for bulk emails. With these changes, an instructor can
include keywords in their bulk emails which will be automatically substituted
with the corresponding value for the recepient of the email. Keywords are
of the form %%keyword%%, and the keywords implemented in this commit include:

%%USER_ID%% => anonymous_user_id
%%USER_FULLNAME%% => user profile name
%%COURSE_DISPLAY_NAME%% => display name of the course
%%COURSE_END_DATE%% => end date of the course

Client-side validations have also been implemented to ensure that only emails
with well-formed keywords can be sent.
The architecture is designed such that adding in new keywords in the future
would be relatively straight-forward.

common/djangoapps/util/keyword_substitution.py

@@ -0,0 +1,82 @@
+"""
+keyword_substitution.py
+
+Contains utility functions to help substitute keywords in a text body with
+the appropriate user / course data.
+
+Supported:
+    LMS:
+        - %%USER_ID%% => anonymous user id
+        - %%USER_FULLNAME%% => User's full name
+        - %%COURSE_DISPLAY_NAME%% => display name of the course
+        - %%COURSE_END_DATE%% => end date of the course
+
+Usage:
+    KEYWORD_FUNCTION_MAP must be supplied in startup.py, so that it lives
+    above other modules in the dependency tree and acts like a global var.
+    Then we can call substitute_keywords_with_data where substitution is
+    needed. Currently called in:
+        - LMS: Announcements + Bulk emails
+        - CMS: Not called
+"""
+
+from django.contrib.auth.models import User
+from xmodule.modulestore.django import modulestore
+
+KEYWORD_FUNCTION_MAP = {}
+
+
+def keyword_function_map_is_empty():
+    """
+    Checks if the keyword function map has been filled
+    """
+    return not bool(KEYWORD_FUNCTION_MAP)
+
+
+def add_keyword_function_map(mapping):
+    """
+    Attaches the given keyword-function mapping to the existing one
+    """
+    KEYWORD_FUNCTION_MAP.update(mapping)
+
+
+def substitute_keywords(string, user=None, course=None):
+    """
+    Replaces all %%-encoded words using KEYWORD_FUNCTION_MAP mapping functions
+
+    Iterates through all keywords that must be substituted and replaces
+    them by calling the corresponding functions stored in KEYWORD_FUNCTION_MAP.
+
+    Functions stored in KEYWORD_FUNCTION_MAP must return a replacement string.
+    Also, functions imported from other modules must be wrapped in a
+    new function if they don't take in user_id and course_id. This simplifies
+    the loop below, and reduces the need for piling up if elif else statements
+    when the keyword pool grows.
+    """
+    if user is None or course is None:
+        # Cannot proceed without course and user information
+        return string
+
+    for key, func in KEYWORD_FUNCTION_MAP.iteritems():
+        if key in string:
+            substitutor = func(user, course)
+            string = string.replace(key, substitutor)
+
+    return string
+
+
+def substitute_keywords_with_data(string, user_id=None, course_id=None):
+    """
+    Given user and course ids, replaces all %%-encoded words in the given string
+    """
+
+    # Do not proceed without parameters: Compatibility check with existing tests
+    # that do not supply these parameters
+    if user_id is None or course_id is None:
+        return string
+
+    # Grab user objects
+    user = User.objects.get(id=user_id)
+    course = modulestore().get_course(course_id, depth=0)
+
+    return substitute_keywords(string, user, course)

common/djangoapps/util/tests/fixtures/test_keyword_anonid_sub.json

@@ -0,0 +1,14 @@
+{
+    "standard_test": {
+        "test_string": "This is a test string. sub this: %%USER_ID%% into anon_id",
+        "expected": "This is a test string. sub this: 123456789 into anon_id"
+    },
+    "multiple_subs": {
+        "test_string": "There are a lot of anonymous ids here: %%USER_ID%% %%USER_ID%% %%USER_ID%% %%USER_ID%%",
+        "expected": "There are a lot of anonymous ids here: 123456789 123456789 123456789 123456789"
+    },
+    "sub_with_html": {
+        "test_string": "There is html in this guy <b>%%USER_ID%%</b>",
+        "expected": "There is html in this guy <b>123456789</b>"
+    }
+}

common/djangoapps/util/tests/fixtures/test_keyword_coursename_sub.json

@@ -0,0 +1,14 @@
+{
+    "simple_test": {
+        "test_string": "Course display name: %%COURSE_DISPLAY_NAME%%",
+        "expected": "Course display name: test_course"
+    },
+    "Multiple_test": {
+        "test_string": "We display %%COURSE_DISPLAY_NAME%% a lot here: %%COURSE_DISPLAY_NAME%% %%COURSE_DISPLAY_NAME%%",
+        "expected": "We display test_course a lot here: test_course test_course"
+    },
+    "Html_test": {
+        "test_string": "This string has some html: <h1>%%COURSE_DISPLAY_NAME%%</h1>",
+        "expected": "This string has some html: <h1>test_course</h1>"
+    }
+}

common/djangoapps/util/tests/fixtures/test_keywordsub_multiple_tags.json

@@ -0,0 +1,10 @@
+{
+    "basic_test": {
+        "test_string": "The user with id %%USER_ID%% is named %%USER_FULLNAME%%",
+        "expected": "The user with id 123456789 is named Test User"
+    },
+    "invalid_tags": {
+        "test_string": "The user with id %%user-id%% is named %%USER_FULLNAME%% and is in %%COURSE_DISPLAY_NAME",
+        "expected": "The user with id %%user-id%% is named Test User and is in test_course"
+    }
+}

common/djangoapps/util/tests/test_keyword_sub_utils.py

@@ -0,0 +1,152 @@
+"""
+Tests for keyword_substitution.py
+"""
+
+from student.tests.factories import UserFactory
+from xmodule.modulestore.tests.factories import CourseFactory
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+from ddt import ddt, file_data
+from mock import patch
+
+
+from util.date_utils import get_default_time_display
+from util import keyword_substitution as Ks
+
+
+@ddt
+class KeywordSubTest(ModuleStoreTestCase):
+    """ Tests for the keyword substitution feature """
+
+    def setUp(self):
+        self.user = UserFactory.create(
+            email="testuser@edx.org",
+            username="testuser",
+            profile__name="Test User"
+        )
+        self.course = CourseFactory.create(
+            org='edx',
+            course='999',
+            display_name='test_course'
+        )
+
+        # Mimic monkeypatching done in startup.py
+        Ks.KEYWORD_FUNCTION_MAP = self.get_keyword_function_map()
+
+    def get_keyword_function_map(self):
+        """
+        Generates a mapping from keywords to functions for testing
+
+        The keyword sub functions should not return %% encoded strings. This
+        would lead to ugly and inconsistent behavior due to the way in which
+        keyword subbing is handled.
+        """
+
+        def user_fullname_sub(user, course):  # pylint: disable=unused-argument
+            """ Returns the user's name """
+            return user.profile.name
+
+        def course_display_name_sub(user, course):  # pylint: disable=unused-argument
+            """ Returns the course name """
+            return course.display_name
+
+        return {
+            '%%USER_FULLNAME%%': user_fullname_sub,
+            '%%COURSE_DISPLAY_NAME%%': course_display_name_sub,
+        }
+
+    @file_data('fixtures/test_keyword_coursename_sub.json')
+    def test_course_name_sub(self, test_info):
+        """ Tests subbing course name in various scenarios """
+        course_name = self.course.display_name
+        result = Ks.substitute_keywords_with_data(test_info['test_string'], self.user.id, self.course.id)
+
+        self.assertIn(course_name, result)
+        self.assertEqual(result, test_info['expected'])
+
+    @file_data('fixtures/test_keyword_anonid_sub.json')
+    def test_anonymous_id_subs(self, test_info):
+        """ Tests subbing anon user id in various scenarios """
+        anon_id = '123456789'
+        with patch.dict(Ks.KEYWORD_FUNCTION_MAP, {'%%USER_ID%%': lambda x, y: anon_id}):
+            result = Ks.substitute_keywords_with_data(test_info['test_string'], self.user.id, self.course.id)
+
+            self.assertIn(anon_id, result)
+            self.assertEqual(result, test_info['expected'])
+
+    def test_name_sub(self):
+        test_string = "This is the test string. subthis:  %%USER_FULLNAME%% into user name"
+        user_name = self.user.profile.name
+        result = Ks.substitute_keywords_with_data(test_string, self.user.id, self.course.id)
+
+        self.assertNotIn('%%USER_FULLNAME%%', result)
+        self.assertIn(user_name, result)
+
+    def test_illegal_subtag(self):
+        """
+        Test that sub-ing doesn't ocurr with illegal tags
+        """
+        test_string = "%%user_id%%"
+        result = Ks.substitute_keywords_with_data(test_string, self.user.id, self.course.id)
+
+        self.assertEquals(test_string, result)
+
+    def test_should_not_sub(self):
+        """
+        Test that sub-ing doesn't work without subtags
+        """
+        test_string = "this string has no subtags"
+        result = Ks.substitute_keywords_with_data(test_string, self.user.id, self.course.id)
+
+        self.assertEquals(test_string, result)
+
+    @file_data('fixtures/test_keywordsub_multiple_tags.json')
+    def test_sub_mutiltple_tags(self, test_info):
+        """ Test that subbing works with multiple subtags """
+        anon_id = '123456789'
+        patched_dict = {
+            '%%USER_ID%%': lambda x, y: anon_id,
+            '%%USER_FULLNAME%%': lambda x, y: self.user.profile.name,
+            '%%COURSE_DISPLAY_NAME': lambda x, y: self.course.display_name,
+            '%%COURSE_END_DATE': lambda x, y: get_default_time_display(self.course.end)
+        }
+
+        with patch.dict(Ks.KEYWORD_FUNCTION_MAP, patched_dict):
+            result = Ks.substitute_keywords_with_data(test_info['test_string'], self.user.id, self.course.id)
+            self.assertEqual(result, test_info['expected'])
+
+    def test_no_subbing_empty_subtable(self):
+        """
+        Test that no sub-ing occurs when the sub table is empty.
+        """
+        # Set the keyword sub mapping to be empty
+        Ks.KEYWORD_FUNCTION_MAP = {}
+
+        test_string = 'This user\'s name is %%USER_FULLNAME%%'
+        result = Ks.substitute_keywords_with_data(test_string, self.user.id, self.course.id)
+
+        self.assertNotIn(self.user.profile.name, result)
+        self.assertIn('%%USER_FULLNAME%%', result)
+
+    def test_subbing_no_userid_or_courseid(self):
+        """
+        Tests that no subbing occurs if no user_id or no course_id is given.
+        """
+        test_string = 'This string should not be subbed here %%USER_ID%%'
+
+        result = Ks.substitute_keywords_with_data(test_string, None, self.course.id)
+        self.assertEqual(test_string, result)
+
+        result = Ks.substitute_keywords_with_data(test_string, self.user.id, None)
+        self.assertEqual(test_string, result)
+
+    def test_subbing_no_user_or_course(self):
+        """
+        Tests that no subbing occurs if no user or no course is given
+        """
+        test_string = "This string should not be subbed here %%USER_ID%%"
+
+        result = Ks.substitute_keywords(test_string, course=self.course, user=None)
+        self.assertEqual(test_string, result)
+
+        result = Ks.substitute_keywords(test_string, self.user, None)
+        self.assertEqual(test_string, result)