diff --git a/common/djangoapps/util/models.py b/common/djangoapps/util/models.py index 6b20219993..f2fb241a82 100644 --- a/common/djangoapps/util/models.py +++ b/common/djangoapps/util/models.py @@ -1 +1,2 @@ # Create your models here. + diff --git a/lms/djangoapps/linkedin/management/commands/__init__.py b/lms/djangoapps/linkedin/management/commands/__init__.py index af1b51d4f3..3696e8e28f 100644 --- a/lms/djangoapps/linkedin/management/commands/__init__.py +++ b/lms/djangoapps/linkedin/management/commands/__init__.py @@ -8,11 +8,14 @@ import uuid from django.conf import settings from django.core.management.base import CommandError +import requests from ...models import LinkedInToken +class LinkedInError(Exception): + pass -class LinkedinAPI(object): +class LinkedInAPI(object): """ Encapsulates the LinkedIn API. """ @@ -74,9 +77,14 @@ class LinkedinAPI(object): """ Make an HTTP call to the LinkedIn JSON API. """ + if settings.LINKEDIN_API.get('TEST_MODE'): + raise LinkedInError( + "Attempting to make real API call while in test mode - " + "Mock LinkedInAPI.call_json_api instead." + ) try: request = urllib2.Request(url, headers={'x-li-format': 'json'}) - response = urllib2.urlopen(request).read() + response = urllib2.urlopen(request, timeout=5).read() return json.loads(response) except urllib2.HTTPError, error: self.http_error(error, "Error calling LinkedIn API") diff --git a/lms/djangoapps/linkedin/management/commands/linkedin_findusers.py b/lms/djangoapps/linkedin/management/commands/linkedin_findusers.py index 520b89168e..95ea23b5a3 100644 --- a/lms/djangoapps/linkedin/management/commands/linkedin_findusers.py +++ b/lms/djangoapps/linkedin/management/commands/linkedin_findusers.py @@ -12,13 +12,14 @@ from django.utils import timezone from optparse import make_option -from ...models import LinkedIn -from . import LinkedinAPI +from util.query import use_read_replica_if_available +from linkedin.models import LinkedIn +from . import LinkedInAPI FRIDAY = 4 -def get_call_limits(): +def get_call_limits(force_unlimited=False): """ Returns a tuple of: (max_checks, checks_per_call, time_between_calls) @@ -40,7 +41,7 @@ def get_call_limits(): lastfriday -= datetime.timedelta(days=1) safeharbor_begin = lastfriday.replace(hour=18, minute=0) safeharbor_end = safeharbor_begin + datetime.timedelta(days=2, hours=11) - if safeharbor_begin < now < safeharbor_end: + if force_unlimited or (safeharbor_begin < now < safeharbor_end): return -1, 80, 1 elif now.hour >= 18 or now.hour < 5: return 500, 80, 1 @@ -62,33 +63,38 @@ class Command(BaseCommand): dest='recheck', default=False, help='Check users that have been checked in the past to see if ' - 'they have joined or left LinkedIn since the last check'), + 'they have joined or left LinkedIn since the last check' + ), make_option( '--force', action='store_true', dest='force', default=False, help='Disregard the parameters provided by LinkedIn about when it ' - 'is appropriate to make API calls.')) + 'is appropriate to make API calls.' + ) + ) def handle(self, *args, **options): """ Check users. """ - api = LinkedinAPI(self) - recheck = options.pop('recheck', False) - force = options.pop('force', False) - if force: - max_checks, checks_per_call, time_between_calls = -1, 80, 1 - else: - max_checks, checks_per_call, time_between_calls = get_call_limits() - if not max_checks: - raise CommandError("No checks allowed during this time.") + api = LinkedInAPI(self) + recheck = options.get('recheck', False) + force = options.get('force', False) + max_checks, checks_per_call, time_between_calls = get_call_limits(force) - def batch_users(): - "Generator to lazily generate batches of users to query." + if not max_checks: + raise CommandError("No checks allowed during this time.") + + def user_batches_to_check(): + """Generate batches of users we should query against LinkedIn.""" count = 0 batch = [] + + users = use_read_replica_if_available( + None + ) for user in User.objects.all(): if not hasattr(user, 'linkedin'): LinkedIn(user=user).save() @@ -98,8 +104,9 @@ class Command(BaseCommand): if len(batch) == checks_per_call: yield batch batch = [] + count += 1 - if max_checks != 1 and count == max_checks: + if max_checks != -1 and count >= max_checks: self.stderr.write( "WARNING: limited to checking only %d users today." % max_checks) @@ -107,20 +114,21 @@ class Command(BaseCommand): if batch: yield batch - def do_batch(batch): - "Process a batch of users." - emails = (u.email for u in batch) - for user, has_account in zip(batch, api.batch(emails)): + def update_linkedin_account_status(users): + """ + Given a an iterable of User objects, check their email addresses + to see if they have LinkedIn email addresses and save that + information to our database. + """ + emails = (u.email for u in users) + for user, has_account in zip(users, api.batch(emails)): linkedin = user.linkedin if linkedin.has_linkedin_account != has_account: linkedin.has_linkedin_account = has_account linkedin.save() - batches = batch_users() - try: - do_batch(batches.next()) # may raise StopIteration - for batch in batches: + for i, user_batch in enumerate(user_batches_to_check()): + if i > 0: + # Sleep between LinkedIn API web service calls time.sleep(time_between_calls) - do_batch(batch) - except StopIteration: - pass + update_linkedin_account_status(user_batch) diff --git a/lms/djangoapps/linkedin/management/commands/linkedin_login.py b/lms/djangoapps/linkedin/management/commands/linkedin_login.py index b2df0112c3..4ec13dd9e2 100644 --- a/lms/djangoapps/linkedin/management/commands/linkedin_login.py +++ b/lms/djangoapps/linkedin/management/commands/linkedin_login.py @@ -3,7 +3,7 @@ Log into LinkedIn API. """ from django.core.management.base import BaseCommand -from . import LinkedinAPI +from . import LinkedInAPI class Command(BaseCommand): @@ -19,7 +19,7 @@ class Command(BaseCommand): def handle(self, *args, **options): """ """ - api = LinkedinAPI(self) + api = LinkedInAPI(self) print "Let's log into your LinkedIn account." print "Start by visiting this url:" print api.authorization_url() diff --git a/lms/djangoapps/linkedin/management/commands/linkedin_mailusers.py b/lms/djangoapps/linkedin/management/commands/linkedin_mailusers.py index 1304d71878..9707e343f6 100644 --- a/lms/djangoapps/linkedin/management/commands/linkedin_mailusers.py +++ b/lms/djangoapps/linkedin/management/commands/linkedin_mailusers.py @@ -16,7 +16,7 @@ from certificates.models import GeneratedCertificate from courseware.courses import get_course_by_id from ...models import LinkedIn -from . import LinkedinAPI +from . import LinkedInAPI class Command(BaseCommand): @@ -43,7 +43,7 @@ class Command(BaseCommand): def __init__(self): super(Command, self).__init__() - self.api = LinkedinAPI(self) + self.api = LinkedInAPI(self) def handle(self, *args, **options): whitelist = self.api.config.get('EMAIL_WHITELIST') diff --git a/lms/djangoapps/linkedin/management/commands/tests/test_api.py b/lms/djangoapps/linkedin/management/commands/tests/test_api.py index 8dc2915a48..62d2d3eab1 100644 --- a/lms/djangoapps/linkedin/management/commands/tests/test_api.py +++ b/lms/djangoapps/linkedin/management/commands/tests/test_api.py @@ -4,11 +4,11 @@ import StringIO from django.core.management.base import CommandError from django.test import TestCase -from linkedin.management.commands import LinkedinAPI +from linkedin.management.commands import LinkedInAPI from linkedin.models import LinkedInToken -class LinkedinAPITests(TestCase): +class LinkedInAPITests(TestCase): def setUp(self): patcher = mock.patch('linkedin.management.commands.uuid.uuid4') @@ -17,7 +17,7 @@ class LinkedinAPITests(TestCase): self.addCleanup(patcher.stop) def make_one(self): - return LinkedinAPI(DummyCommand()) + return LinkedInAPI(DummyCommand()) @mock.patch('django.conf.settings.LINKEDIN_API', None) def test_ctor_no_api_config(self): diff --git a/lms/djangoapps/linkedin/management/commands/tests/test_findusers.py b/lms/djangoapps/linkedin/management/commands/tests/test_findusers.py index 984ee5cd08..ae05c8161c 100644 --- a/lms/djangoapps/linkedin/management/commands/tests/test_findusers.py +++ b/lms/djangoapps/linkedin/management/commands/tests/test_findusers.py @@ -69,7 +69,7 @@ class FindUsersTests(TestCase): @mock.patch(MODULE + 'time') @mock.patch(MODULE + 'User') - @mock.patch(MODULE + 'LinkedinAPI') + @mock.patch(MODULE + 'LinkedInAPI') @mock.patch(MODULE + 'get_call_limits') def test_command_success_recheck_no_limits(self, get_call_limits, apicls, usercls, time): @@ -93,7 +93,7 @@ class FindUsersTests(TestCase): @mock.patch(MODULE + 'time') @mock.patch(MODULE + 'User') - @mock.patch(MODULE + 'LinkedinAPI') + @mock.patch(MODULE + 'LinkedInAPI') @mock.patch(MODULE + 'get_call_limits') def test_command_success_no_recheck_no_limits(self, get_call_limits, apicls, usercls, time): @@ -123,7 +123,7 @@ class FindUsersTests(TestCase): @mock.patch(MODULE + 'time') @mock.patch(MODULE + 'User') - @mock.patch(MODULE + 'LinkedinAPI') + @mock.patch(MODULE + 'LinkedInAPI') @mock.patch(MODULE + 'get_call_limits') def test_command_success_no_recheck_no_users(self, get_call_limits, apicls, usercls, time): @@ -149,7 +149,7 @@ class FindUsersTests(TestCase): @mock.patch(MODULE + 'time') @mock.patch(MODULE + 'User') - @mock.patch(MODULE + 'LinkedinAPI') + @mock.patch(MODULE + 'LinkedInAPI') @mock.patch(MODULE + 'get_call_limits') def test_command_success_recheck_with_limit(self, get_call_limits, apicls, usercls, time): @@ -178,7 +178,7 @@ class FindUsersTests(TestCase): self.assertTrue(command.stderr.getvalue().startswith("WARNING")) @mock.patch(MODULE + 'User') - @mock.patch(MODULE + 'LinkedinAPI') + @mock.patch(MODULE + 'LinkedInAPI') @mock.patch(MODULE + 'get_call_limits') def test_command_success_recheck_with_force(self, get_call_limits, apicls, usercls): @@ -199,6 +199,7 @@ class FindUsersTests(TestCase): "Mock LinkedIn API." return [email % 2 == 0 for email in emails] api.batch = dummy_batch + get_call_limits.return_value = (-1, 80, 1) fut(force=True) self.assertEqual([u.linkedin.has_linkedin_account for u in users], [i % 2 == 0 for i in xrange(10)]) diff --git a/lms/envs/test.py b/lms/envs/test.py index 3ec4aaa158..a5d8b182ff 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -266,6 +266,7 @@ LINKEDIN_API = { 'COMPANY_NAME': 'edX', 'COMPANY_ID': '0000000', 'EMAIL_FROM': 'The Team ', + 'TEST_MODE': True } ################### Make tests faster