diff --git a/common/djangoapps/terrain/stubs/catalog.py b/common/djangoapps/terrain/stubs/catalog.py index f33075a774..543db6ec7a 100644 --- a/common/djangoapps/terrain/stubs/catalog.py +++ b/common/djangoapps/terrain/stubs/catalog.py @@ -2,8 +2,11 @@ Stub implementation of catalog service for acceptance tests """ # pylint: disable=invalid-name, missing-docstring +from __future__ import absolute_import + import re -import urlparse + +import six.moves.urllib.parse # pylint: disable=import-error from .http import StubHttpRequestHandler, StubHttpService @@ -27,7 +30,7 @@ class StubCatalogServiceHandler(StubHttpRequestHandler): """ Find the correct handler method given the path info from the HTTP request. """ - path = urlparse.urlparse(self.path).path + path = six.moves.urllib.parse.urlparse(self.path).path for pattern, handler in pattern_handlers.items(): match = re.match(pattern, path) if match: diff --git a/common/djangoapps/terrain/stubs/comments.py b/common/djangoapps/terrain/stubs/comments.py index dfaf01b6c6..702926f919 100644 --- a/common/djangoapps/terrain/stubs/comments.py +++ b/common/djangoapps/terrain/stubs/comments.py @@ -2,8 +2,11 @@ Stub implementation of cs_comments_service for acceptance tests """ +from __future__ import absolute_import + import re -import urlparse + +import six.moves.urllib.parse # pylint: disable=import-error from .http import StubHttpRequestHandler, StubHttpService @@ -12,7 +15,7 @@ class StubCommentsServiceHandler(StubHttpRequestHandler): @property def _params(self): - return urlparse.parse_qs(urlparse.urlparse(self.path).query) + return six.moves.urllib.parse.parse_qs(six.moves.urllib.parse.urlparse(self.path).query) def do_GET(self): pattern_handlers = { @@ -30,7 +33,7 @@ class StubCommentsServiceHandler(StubHttpRequestHandler): self.send_response(404, content="404 Not Found") def match_pattern(self, pattern_handlers): - path = urlparse.urlparse(self.path).path + path = six.moves.urllib.parse.urlparse(self.path).path for pattern in pattern_handlers: match = re.match(pattern, path) if match: @@ -94,7 +97,7 @@ class StubCommentsServiceHandler(StubHttpRequestHandler): def do_thread(self, thread_id): if thread_id in self.server.config.get('threads', {}): thread = self.server.config['threads'][thread_id].copy() - params = urlparse.parse_qs(urlparse.urlparse(self.path).query) + params = six.moves.urllib.parse.parse_qs(six.moves.urllib.parse.urlparse(self.path).query) if "recursive" in params and params["recursive"][0] == "True": thread.setdefault('children', []) resp_total = thread.setdefault('resp_total', len(thread['children'])) @@ -107,7 +110,7 @@ class StubCommentsServiceHandler(StubHttpRequestHandler): def do_threads(self): threads = self.server.config.get('threads', {}) - threads_data = threads.values() + threads_data = list(threads.values()) self.send_json_response({"collection": threads_data, "page": 1, "num_pages": 1}) def do_search_threads(self): diff --git a/common/djangoapps/terrain/stubs/ecommerce.py b/common/djangoapps/terrain/stubs/ecommerce.py index 54a7fa1499..35c86fee31 100644 --- a/common/djangoapps/terrain/stubs/ecommerce.py +++ b/common/djangoapps/terrain/stubs/ecommerce.py @@ -2,8 +2,11 @@ Stub implementation of ecommerce service for acceptance tests """ +from __future__ import absolute_import + import re -import urlparse + +import six.moves.urllib.parse # pylint: disable=import-error from .http import StubHttpRequestHandler, StubHttpService @@ -22,7 +25,7 @@ class StubEcommerceServiceHandler(StubHttpRequestHandler): # pylint: disable=mi """ Find the correct handler method given the path info from the HTTP request. """ - path = urlparse.urlparse(self.path).path + path = six.moves.urllib.parse.urlparse(self.path).path for pattern in pattern_handlers: match = re.match(pattern, path) if match: diff --git a/common/djangoapps/terrain/stubs/edxnotes.py b/common/djangoapps/terrain/stubs/edxnotes.py index 0a180ffc6e..027fdd23ea 100644 --- a/common/djangoapps/terrain/stubs/edxnotes.py +++ b/common/djangoapps/terrain/stubs/edxnotes.py @@ -2,14 +2,18 @@ Stub implementation of EdxNotes for acceptance tests """ +from __future__ import absolute_import + import json import re from copy import deepcopy from datetime import datetime from math import ceil -from urllib import urlencode from uuid import uuid4 +import six +from six.moves.urllib.parse import urlencode # pylint: disable=import-error + from .http import StubHttpRequestHandler, StubHttpService @@ -390,4 +394,4 @@ class StubEdxNotesService(StubHttpService): """ Search the `query(str)` text in the provided `data(list)`. """ - return [note for note in data if unicode(query).strip() in note.get("text", "").split()] + return [note for note in data if six.text_type(query).strip() in note.get("text", "").split()] diff --git a/common/djangoapps/terrain/stubs/http.py b/common/djangoapps/terrain/stubs/http.py index 11fab8a753..dfd2902ca5 100644 --- a/common/djangoapps/terrain/stubs/http.py +++ b/common/djangoapps/terrain/stubs/http.py @@ -2,16 +2,20 @@ Stub implementation of an HTTP service. """ +from __future__ import absolute_import + import json import threading -import urllib -import urlparse -from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer from functools import wraps from logging import getLogger -from SocketServer import ThreadingMixIn +import six +import six.moves.urllib.error # pylint: disable=import-error +import six.moves.urllib.parse # pylint: disable=import-error +import six.moves.urllib.request # pylint: disable=import-error from lazy import lazy +from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer # pylint: disable=import-error +from six.moves.socketserver import ThreadingMixIn # pylint: disable=import-error LOGGER = getLogger(__name__) @@ -107,7 +111,7 @@ class StubHttpRequestHandler(BaseHTTPRequestHandler, object): # None of our parameters are lists, however, so we map [val] --> val # If the list contains multiple entries, we pick the first one try: - post_dict = urlparse.parse_qs(contents, keep_blank_values=True) + post_dict = six.moves.urllib.parse.parse_qs(contents, keep_blank_values=True) return { key: list_val[0] for key, list_val in post_dict.items() @@ -121,13 +125,13 @@ class StubHttpRequestHandler(BaseHTTPRequestHandler, object): """ Return the GET parameters (querystring in the URL). """ - query = urlparse.urlparse(self.path).query + query = six.moves.urllib.parse.urlparse(self.path).query # By default, `parse_qs` returns a list of values for each param # For convenience, we replace lists of 1 element with just the element return { key: value[0] if len(value) == 1 else value - for key, value in urlparse.parse_qs(query).items() + for key, value in six.moves.urllib.parse.parse_qs(query).items() } @lazy @@ -136,7 +140,7 @@ class StubHttpRequestHandler(BaseHTTPRequestHandler, object): Return the URL path without GET parameters. Removes the trailing slash if there is one. """ - path = urlparse.urlparse(self.path).path + path = six.moves.urllib.parse.urlparse(self.path).path if path.endswith('/'): return path[:-1] else: @@ -153,12 +157,12 @@ class StubHttpRequestHandler(BaseHTTPRequestHandler, object): if self.path == "/set_config" or self.path == "/set_config/": if len(self.post_dict) > 0: - for key, value in self.post_dict.iteritems(): + for key, value in six.iteritems(self.post_dict): # Decode the params as UTF-8 try: - key = unicode(key, 'utf-8') - value = unicode(value, 'utf-8') + key = six.text_type(key, 'utf-8') + value = six.text_type(value, 'utf-8') except UnicodeDecodeError: self.log_message("Could not decode request params as UTF-8") @@ -221,7 +225,7 @@ class StubHttpRequestHandler(BaseHTTPRequestHandler, object): `args` is an array of values to fill into the string. """ if not args: - format_str = urllib.unquote(format_str) + format_str = six.moves.urllib.parse.unquote(format_str) return u"{0} - - [{1}] {2}\n".format( self.client_address[0], self.log_date_time_string(), diff --git a/common/djangoapps/terrain/stubs/lti.py b/common/djangoapps/terrain/stubs/lti.py index 5d9cc6f6db..fd4f2cfed2 100644 --- a/common/djangoapps/terrain/stubs/lti.py +++ b/common/djangoapps/terrain/stubs/lti.py @@ -9,20 +9,27 @@ not possible to have this LTI multiple times on a single page in LMS. """ +from __future__ import absolute_import + import base64 import hashlib +import logging import os import textwrap -import urllib from uuid import uuid4 + import mock import oauthlib.oauth1 import requests -from openedx.core.djangolib.markup import HTML -from http import StubHttpRequestHandler, StubHttpService +import six +import six.moves.urllib.error # pylint: disable=import-error +import six.moves.urllib.parse # pylint: disable=import-error +import six.moves.urllib.request # pylint: disable=import-error from oauthlib.oauth1.rfc5849 import parameters, signature -import logging +from openedx.core.djangolib.markup import HTML + +from .http import StubHttpRequestHandler, StubHttpService log = logging.getLogger(__name__) @@ -233,7 +240,7 @@ class StubLtiHandler(StubHttpRequestHandler): # Currently LTI module doublequotes the lis_result_sourcedid parameter. # Unquote response two times. - return urllib.unquote(urllib.unquote(response_str)) + return six.moves.urllib.parse.unquote(six.moves.urllib.parse.unquote(response_str)) def _is_correct_lti_request(self): """ @@ -249,8 +256,8 @@ class StubLtiHandler(StubHttpRequestHandler): client_key = self.server.config.get('client_key', self.DEFAULT_CLIENT_KEY) client_secret = self.server.config.get('client_secret', self.DEFAULT_CLIENT_SECRET) client = oauthlib.oauth1.Client( - client_key=unicode(client_key), - client_secret=unicode(client_secret) + client_key=six.text_type(client_key), + client_secret=six.text_type(client_secret) ) headers = { # This is needed for body encoding: @@ -260,13 +267,13 @@ class StubLtiHandler(StubHttpRequestHandler): # Calculate and encode body hash. See http://oauth.googlecode.com/svn/spec/ext/body_hash/1.0/oauth-bodyhash.html sha1 = hashlib.sha1() sha1.update(body) - oauth_body_hash = unicode(base64.b64encode(sha1.digest())) + oauth_body_hash = six.text_type(base64.b64encode(sha1.digest())) mock_request = mock.Mock( - uri=unicode(urllib.unquote(url)), + uri=six.text_type(six.moves.urllib.parse.unquote(url)), headers=headers, body=u"", decoded_body=u"", - http_method=unicode(method), + http_method=six.text_type(method), ) params = client.get_oauth_params(mock_request) mock_request.oauth_params = params @@ -292,17 +299,17 @@ class StubLtiHandler(StubHttpRequestHandler): Returns `True` if signatures are correct, otherwise `False`. """ - client_secret = unicode(self.server.config.get('client_secret', self.DEFAULT_CLIENT_SECRET)) + client_secret = six.text_type(self.server.config.get('client_secret', self.DEFAULT_CLIENT_SECRET)) host = os.environ.get('BOK_CHOY_HOSTNAME', '127.0.0.1') port = self.server.server_address[1] lti_base = self.DEFAULT_LTI_ADDRESS.format(host=host, port=port) lti_endpoint = self.server.config.get('lti_endpoint', self.DEFAULT_LTI_ENDPOINT) url = lti_base + lti_endpoint request = mock.Mock() - request.params = [(unicode(k), unicode(v)) for k, v in params.items()] - request.uri = unicode(url) + request.params = [(six.text_type(k), six.text_type(v)) for k, v in params.items()] + request.uri = six.text_type(url) request.http_method = u'POST' - request.signature = unicode(client_signature) + request.signature = six.text_type(client_signature) return signature.verify_hmac_sha1(request, client_secret) diff --git a/common/djangoapps/terrain/stubs/start.py b/common/djangoapps/terrain/stubs/start.py index c76859fa4d..7827379240 100644 --- a/common/djangoapps/terrain/stubs/start.py +++ b/common/djangoapps/terrain/stubs/start.py @@ -1,7 +1,8 @@ """ Command-line utility to start a stub service. """ -from __future__ import print_function +from __future__ import absolute_import, print_function + import logging import sys import time @@ -47,7 +48,7 @@ def get_args(): if service_name not in SERVICES: print("Unrecognized service '{0}'. Valid choices are: {1}".format( - service_name, ", ".join(SERVICES.keys()))) + service_name, ", ".join(list(SERVICES.keys())))) sys.exit(1) try: diff --git a/common/djangoapps/terrain/stubs/tests/test_edxnotes.py b/common/djangoapps/terrain/stubs/tests/test_edxnotes.py index 9b6a464f24..32e1e66d22 100644 --- a/common/djangoapps/terrain/stubs/tests/test_edxnotes.py +++ b/common/djangoapps/terrain/stubs/tests/test_edxnotes.py @@ -1,12 +1,17 @@ """ Unit tests for stub EdxNotes implementation. """ -import ddt -import urlparse +from __future__ import absolute_import + import json import unittest -import requests from uuid import uuid4 + +import ddt +import requests +import six.moves.urllib.parse # pylint: disable=import-error +from six.moves import range + from ..edxnotes import StubEdxNotesService @@ -29,7 +34,7 @@ class StubEdxNotesServiceTest(unittest.TestCase): """ Returns a list of dummy notes. """ - return [self._get_dummy_note(i) for i in xrange(count)] # pylint: disable=unused-variable + return [self._get_dummy_note(i) for i in range(count)] # pylint: disable=unused-variable def _get_dummy_note(self, uid=0): """ @@ -157,8 +162,8 @@ class StubEdxNotesServiceTest(unittest.TestCase): }) self.assertTrue(response.ok) response = response.json() - parsed = urlparse.urlparse(url) - query_params = urlparse.parse_qs(parsed.query) + parsed = six.moves.urllib.parse.urlparse(url) + query_params = six.moves.urllib.parse.parse_qs(parsed.query) query_params['usage_id'].reverse() self.assertEqual(len(response), len(query_params['usage_id'])) for index, usage_id in enumerate(query_params['usage_id']): @@ -224,8 +229,8 @@ class StubEdxNotesServiceTest(unittest.TestCase): if url is None: return None - parsed = urlparse.urlparse(url) - query_params = urlparse.parse_qs(parsed.query) + parsed = six.moves.urllib.parse.urlparse(url) + query_params = six.moves.urllib.parse.parse_qs(parsed.query) page = query_params["page"][0] return page if page is None else int(page) diff --git a/common/djangoapps/terrain/stubs/tests/test_http.py b/common/djangoapps/terrain/stubs/tests/test_http.py index ee3dfae78b..f9ab9be06f 100644 --- a/common/djangoapps/terrain/stubs/tests/test_http.py +++ b/common/djangoapps/terrain/stubs/tests/test_http.py @@ -2,10 +2,15 @@ Unit tests for stub HTTP server base class. """ -import unittest -import requests +from __future__ import absolute_import + import json -from terrain.stubs.http import StubHttpService, StubHttpRequestHandler, require_params +import unittest + +import requests +import six + +from terrain.stubs.http import StubHttpRequestHandler, StubHttpService, require_params class StubHttpServiceTest(unittest.TestCase): @@ -35,7 +40,7 @@ class StubHttpServiceTest(unittest.TestCase): 'test_boolean': False } - for key, val in params.iteritems(): + for key, val in six.iteritems(params): # JSON-encode each parameter post_params = {key: json.dumps(val)} @@ -43,7 +48,7 @@ class StubHttpServiceTest(unittest.TestCase): self.assertEqual(response.status_code, 200) # Check that the expected values were set in the configuration - for key, val in params.iteritems(): + for key, val in six.iteritems(params): self.assertEqual(self.server.config.get(key), val) def test_bad_json(self): diff --git a/common/djangoapps/terrain/stubs/tests/test_lti_stub.py b/common/djangoapps/terrain/stubs/tests/test_lti_stub.py index c9204a299b..43e3dc0044 100644 --- a/common/djangoapps/terrain/stubs/tests/test_lti_stub.py +++ b/common/djangoapps/terrain/stubs/tests/test_lti_stub.py @@ -1,10 +1,14 @@ """ Unit tests for stub LTI implementation. """ -from mock import Mock, patch +from __future__ import absolute_import + import unittest -import urllib2 + import requests +import six.moves.urllib.request # pylint: disable=import-error +from mock import Mock, patch + from terrain.stubs.lti import StubLtiService @@ -70,7 +74,7 @@ class StubLtiServiceTest(unittest.TestCase): grade_uri = self.uri + 'grade' with patch('terrain.stubs.lti.requests.post') as mocked_post: mocked_post.return_value = Mock(content='Test response', status_code=200) - response = urllib2.urlopen(grade_uri, data='') + response = six.moves.urllib.request.urlopen(grade_uri, data='') self.assertIn('Test response', response.read()) @patch('terrain.stubs.lti.signature.verify_hmac_sha1', return_value=True) @@ -80,7 +84,7 @@ class StubLtiServiceTest(unittest.TestCase): grade_uri = self.uri + 'lti2_outcome' with patch('terrain.stubs.lti.requests.put') as mocked_put: mocked_put.return_value = Mock(status_code=200) - response = urllib2.urlopen(grade_uri, data='') + response = six.moves.urllib.request.urlopen(grade_uri, data='') self.assertIn('LTI consumer (edX) responded with HTTP 200', response.read()) @patch('terrain.stubs.lti.signature.verify_hmac_sha1', return_value=True) @@ -90,5 +94,5 @@ class StubLtiServiceTest(unittest.TestCase): grade_uri = self.uri + 'lti2_delete' with patch('terrain.stubs.lti.requests.put') as mocked_put: mocked_put.return_value = Mock(status_code=200) - response = urllib2.urlopen(grade_uri, data='') + response = six.moves.urllib.request.urlopen(grade_uri, data='') self.assertIn('LTI consumer (edX) responded with HTTP 200', response.read()) diff --git a/common/djangoapps/terrain/stubs/tests/test_video.py b/common/djangoapps/terrain/stubs/tests/test_video.py index de7324bf4d..7729f59c74 100644 --- a/common/djangoapps/terrain/stubs/tests/test_video.py +++ b/common/djangoapps/terrain/stubs/tests/test_video.py @@ -1,11 +1,15 @@ """ Unit tests for Video stub server implementation. """ +from __future__ import absolute_import + import unittest + import requests -from terrain.stubs.video_source import VideoSourceHttpService from django.conf import settings +from terrain.stubs.video_source import VideoSourceHttpService + HLS_MANIFEST_TEXT = """ #EXTM3U #EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=264787,RESOLUTION=1280x720 diff --git a/common/djangoapps/terrain/stubs/tests/test_xqueue_stub.py b/common/djangoapps/terrain/stubs/tests/test_xqueue_stub.py index 4187cbfb80..2313044278 100644 --- a/common/djangoapps/terrain/stubs/tests/test_xqueue_stub.py +++ b/common/djangoapps/terrain/stubs/tests/test_xqueue_stub.py @@ -2,10 +2,14 @@ Unit tests for stub XQueue implementation. """ -import mock -import unittest +from __future__ import absolute_import + import json +import unittest + +import mock import requests + from ..xqueue import StubXQueueService diff --git a/common/djangoapps/terrain/stubs/tests/test_youtube_stub.py b/common/djangoapps/terrain/stubs/tests/test_youtube_stub.py index 467d7d7d03..0ad715420e 100644 --- a/common/djangoapps/terrain/stubs/tests/test_youtube_stub.py +++ b/common/djangoapps/terrain/stubs/tests/test_youtube_stub.py @@ -2,8 +2,12 @@ Unit test for stub YouTube implementation. """ +from __future__ import absolute_import + import unittest + import requests + from ..youtube import StubYouTubeService diff --git a/common/djangoapps/terrain/stubs/video_source.py b/common/djangoapps/terrain/stubs/video_source.py index 1fcda039e8..144ce616e3 100644 --- a/common/djangoapps/terrain/stubs/video_source.py +++ b/common/djangoapps/terrain/stubs/video_source.py @@ -1,10 +1,13 @@ """ Serve HTML5 video sources for acceptance tests """ +from __future__ import absolute_import + import os from contextlib import contextmanager from logging import getLogger -from SimpleHTTPServer import SimpleHTTPRequestHandler + +from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler from .http import StubHttpService diff --git a/common/djangoapps/terrain/stubs/xqueue.py b/common/djangoapps/terrain/stubs/xqueue.py index 9cf6889a47..ae535e1814 100644 --- a/common/djangoapps/terrain/stubs/xqueue.py +++ b/common/djangoapps/terrain/stubs/xqueue.py @@ -9,12 +9,17 @@ Configuration values: If no grade response is configured, a default response will be returned. """ +from __future__ import absolute_import + import copy import json from threading import Timer +import six from requests import post +from openedx.core.djangolib.markup import HTML + from .http import StubHttpRequestHandler, StubHttpService, require_params @@ -147,7 +152,7 @@ class StubXQueueHandler(StubHttpRequestHandler): # Wrap the message in