From a0e4e94e31ab6e3fbcb9d52f901e642ec54f7945 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Mon, 16 Sep 2019 12:29:50 +0500 Subject: [PATCH] BOM-482 Fixing python3 --- lms/djangoapps/dashboard/git_import.py | 6 +-- lms/djangoapps/instructor/tests/test_api.py | 59 ++++++++++++--------- lms/djangoapps/instructor/views/api.py | 12 +++-- 3 files changed, 45 insertions(+), 32 deletions(-) diff --git a/lms/djangoapps/dashboard/git_import.py b/lms/djangoapps/dashboard/git_import.py index bfb5379f49..a888f7374c 100644 --- a/lms/djangoapps/dashboard/git_import.py +++ b/lms/djangoapps/dashboard/git_import.py @@ -117,7 +117,7 @@ def cmd_log(cmd, cwd): used along with the output. Will raise subprocess.CalledProcessError if command doesn't return 0, and returns the command's output. """ - output = subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT) + output = subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT).decode('utf-8') log.debug(u'Command was: %r. Working directory was: %r', ' '.join(cmd), cwd) log.debug(u'Command output was: %r', output) @@ -230,7 +230,7 @@ def add_repo(repo, rdir_in, branch=None): cwd = os.path.abspath(cwd) try: - ret_git = cmd_log(cmd, cwd=cwd).decode('utf-8') + ret_git = cmd_log(cmd, cwd=cwd) except subprocess.CalledProcessError as ex: log.exception(u'Error running git pull: %r', ex.output) raise GitImportErrorCannotPull() @@ -246,7 +246,7 @@ def add_repo(repo, rdir_in, branch=None): log.exception(u'Unable to get git log: %r', ex.output) raise GitImportErrorBadRepo() - ret_git += u'\nCommit ID: {0}'.format(commit_id.decode('utf-8')) + ret_git += u'\nCommit ID: {0}'.format(commit_id) # get branch cmd = ['git', 'symbolic-ref', '--short', 'HEAD', ] diff --git a/lms/djangoapps/instructor/tests/test_api.py b/lms/djangoapps/instructor/tests/test_api.py index 69c5669ad6..f37cec6d66 100644 --- a/lms/djangoapps/instructor/tests/test_api.py +++ b/lms/djangoapps/instructor/tests/test_api.py @@ -14,6 +14,7 @@ import tempfile import ddt import pytest +import six from boto.exception import BotoServerError from django.conf import settings from django.contrib.auth.models import User @@ -294,27 +295,30 @@ class TestCommonExceptions400(TestCase): self.request.is_ajax.return_value = False resp = view_user_doesnotexist(self.request) # pylint: disable=assignment-from-no-return self.assertEqual(resp.status_code, 400) - self.assertIn("User does not exist", resp.content) + self.assertIn("User does not exist", resp.content.decode("utf-8")) def test_user_doesnotexist_ajax(self): self.request.is_ajax.return_value = True resp = view_user_doesnotexist(self.request) # pylint: disable=assignment-from-no-return self.assertEqual(resp.status_code, 400) - self.assertIn("User does not exist", resp.content) + self.assertIn("User does not exist", resp.content.decode("utf-8")) @ddt.data(True, False) def test_alreadyrunningerror(self, is_ajax): self.request.is_ajax.return_value = is_ajax resp = view_alreadyrunningerror(self.request) # pylint: disable=assignment-from-no-return self.assertEqual(resp.status_code, 400) - self.assertIn("Requested task is already running", resp.content) + self.assertIn("Requested task is already running", resp.content.decode("utf-8")) @ddt.data(True, False) def test_alreadyrunningerror_with_unicode(self, is_ajax): self.request.is_ajax.return_value = is_ajax resp = view_alreadyrunningerror_unicode(self.request) # pylint: disable=assignment-from-no-return self.assertEqual(resp.status_code, 400) - self.assertIn('Text with unicode chárácters', resp.content) + self.assertIn( + u'Text with unicode chárácters', + resp.content.decode('utf-8') + ) @ddt.data(True, False) def test_queue_connection_error(self, is_ajax): @@ -324,7 +328,10 @@ class TestCommonExceptions400(TestCase): self.request.is_ajax.return_value = is_ajax resp = view_queue_connection_error(self.request) # pylint: disable=assignment-from-no-return self.assertEqual(resp.status_code, 400) - self.assertIn('Error occured. Please try again later', resp.content) + self.assertIn( + 'Error occured. Please try again later', + resp.content.decode('utf-8') + ) @ddt.ddt @@ -696,7 +703,7 @@ class TestInstructorAPIBulkAccountCreationAndEnrollment(SharedModuleStoreTestCas """ Happy path test to create a single new user """ - csv_content = "\ntest_student@example.com,test_student_1,tester1,USA\n\n" + csv_content = b"\ntest_student@example.com,test_student_1,tester1,USA\n\n" uploaded_file = SimpleUploadedFile("temp.csv", csv_content) response = self.client.post(self.url, {'students_list': uploaded_file}) self.assertEqual(response.status_code, 200) @@ -748,7 +755,10 @@ class TestInstructorAPIBulkAccountCreationAndEnrollment(SharedModuleStoreTestCas self.assertEqual(response.status_code, 200) data = json.loads(response.content.decode('utf-8')) self.assertNotEquals(len(data['general_errors']), 0) - self.assertEquals(data['general_errors'][0]['response'], 'Make sure that the file you upload is in CSV format with no extraneous characters or rows.') + self.assertEquals( + data['general_errors'][0]['response'], + 'Make sure that the file you upload is in CSV format with no extraneous characters or rows.' + ) manual_enrollments = ManualEnrollmentAudit.objects.all() self.assertEqual(manual_enrollments.count(), 0) @@ -861,8 +871,7 @@ class TestInstructorAPIBulkAccountCreationAndEnrollment(SharedModuleStoreTestCas user.save() csv_content = b"{email},{username},tester,USA".format(email=conflicting_email, username='new_test_student') - - uploaded_file = SimpleUploadedFile("temp.csv", csv_content) + uploaded_file = SimpleUploadedFile("temp.csv", six.b(csv_content)) response = self.client.post(self.url, {'students_list': uploaded_file}) self.assertEqual(response.status_code, 200) data = json.loads(response.content.decode('utf-8')) @@ -3283,7 +3292,7 @@ class TestInstructorAPILevelsDataDump(SharedModuleStoreTestCase, LoginEnrollment url = reverse('get_anon_ids', kwargs={'course_id': text_type(self.course.id)}) response = self.client.post(url, {}) self.assertEqual(response['Content-Type'], 'text/csv') - body = response.content.replace('\r', '') + body = response.content.decode("utf-8").replace('\r', '') self.assertTrue(body.startswith( '"User ID","Anonymized User ID","Course Specific Anonymized User ID"' '\n"{user_id}","41","42"\n'.format(user_id=self.students[0].id) @@ -4906,7 +4915,7 @@ class TestCourseRegistrationCodes(SharedModuleStoreTestCase): response = self.client.post(url, data, **{'HTTP_HOST': 'localhost'}) self.assertEqual(response.status_code, 200, response.content) self.assertEqual(response['Content-Type'], 'text/csv') - body = response.content.replace('\r', '') + body = response.content.decode('utf-8').replace('\r', '') self.assertTrue(body.startswith(EXPECTED_CSV_HEADER)) self.assertEqual(len(body.split('\n')), 17) @@ -4929,7 +4938,7 @@ class TestCourseRegistrationCodes(SharedModuleStoreTestCase): self.assertEqual(response.status_code, 200, response.content) self.assertEqual(response['Content-Type'], 'text/csv') - body = response.content.replace('\r', '') + body = response.content.decode('utf-8').replace('\r', '') self.assertTrue(body.startswith(EXPECTED_CSV_HEADER)) self.assertEqual(len(body.split('\n')), 17) rows = body.split('\n') @@ -4964,7 +4973,7 @@ class TestCourseRegistrationCodes(SharedModuleStoreTestCase): response = self.client.post(url, data, **{'HTTP_HOST': 'localhost'}) self.assertEqual(response.status_code, 200, response.content) self.assertEqual(response['Content-Type'], 'text/csv') - body = response.content.replace('\r', '') + body = response.content.decode('utf-8').replace('\r', '') self.assertTrue(body.startswith(EXPECTED_CSV_HEADER)) self.assertEqual(len(body.split('\n')), 5) # 1 for headers, 1 for new line at the end and 3 for the actual data @@ -4988,7 +4997,7 @@ class TestCourseRegistrationCodes(SharedModuleStoreTestCase): response = self.client.post(url, data, **{'HTTP_HOST': 'localhost'}) self.assertEqual(response.status_code, 200, response.content) self.assertEqual(response['Content-Type'], 'text/csv') - body = response.content.replace('\r', '') + body = response.content.decode('utf-8').replace('\r', '') self.assertTrue(body.startswith(EXPECTED_CSV_HEADER)) self.assertEqual(len(body.split('\n')), 4) @@ -5003,7 +5012,7 @@ class TestCourseRegistrationCodes(SharedModuleStoreTestCase): response = self.client.post(url, data) self.assertEqual(response.status_code, 200, response.content) self.assertEqual(response['Content-Type'], 'text/csv') - body = response.content.replace('\r', '') + body = response.content.decode('utf-8').replace('\r', '') self.assertTrue(body.startswith(EXPECTED_CSV_HEADER)) @@ -5041,7 +5050,7 @@ class TestCourseRegistrationCodes(SharedModuleStoreTestCase): response = self.client.post(url, data) self.assertEqual(response.status_code, 200, response.content) self.assertEqual(response['Content-Type'], 'text/csv') - body = response.content.replace('\r', '') + body = response.content.decode('utf-8').replace('\r', '') self.assertTrue(body.startswith(EXPECTED_CSV_HEADER)) self.assertEqual(len(body.split('\n')), 11) @@ -5056,7 +5065,7 @@ class TestCourseRegistrationCodes(SharedModuleStoreTestCase): response = self.client.post(url, data) self.assertEqual(response.status_code, 200, response.content) self.assertEqual(response['Content-Type'], 'text/csv') - body = response.content.replace('\r', '') + body = response.content.decode('utf-8').replace('\r', '') self.assertTrue(body.startswith(EXPECTED_CSV_HEADER)) self.assertEqual(len(body.split('\n')), 9) @@ -5079,7 +5088,7 @@ class TestCourseRegistrationCodes(SharedModuleStoreTestCase): response = self.client.post(url, data) self.assertEqual(response.status_code, 200, response.content) self.assertEqual(response['Content-Type'], 'text/csv') - body = response.content.replace('\r', '') + body = response.content.decode('utf-8').replace('\r', '') self.assertTrue(body.startswith(EXPECTED_CSV_HEADER)) self.assertEqual(len(body.split('\n')), 11) @@ -5095,7 +5104,7 @@ class TestCourseRegistrationCodes(SharedModuleStoreTestCase): response = self.client.post(url, data) self.assertEqual(response.status_code, 200, response.content) self.assertEqual(response['Content-Type'], 'text/csv') - body = response.content.replace('\r', '') + body = response.content.decode('utf-8').replace('\r', '') self.assertTrue(body.startswith(EXPECTED_CSV_HEADER)) self.assertEqual(len(body.split('\n')), 14) @@ -5118,7 +5127,7 @@ class TestCourseRegistrationCodes(SharedModuleStoreTestCase): response = self.client.post(url, data) self.assertEqual(response.status_code, 200, response.content) self.assertEqual(response['Content-Type'], 'text/csv') - body = response.content.replace('\r', '') + body = response.content.decode('utf-8').replace('\r', '') self.assertTrue(body.startswith(EXPECTED_CSV_HEADER)) self.assertEqual(len(body.split('\n')), 11) @@ -5166,7 +5175,7 @@ class TestCourseRegistrationCodes(SharedModuleStoreTestCase): response = self.client.post(url, data) self.assertEqual(response.status_code, 200, response.content) self.assertEqual(response['Content-Type'], 'text/csv') - body = response.content.replace('\r', '') + body = response.content.decode('utf-8').replace('\r', '') self.assertTrue(body.startswith(EXPECTED_CSV_HEADER)) def test_with_invalid_unit_price(self): @@ -5186,8 +5195,8 @@ class TestCourseRegistrationCodes(SharedModuleStoreTestCase): } response = self.client.post(generate_code_url, data, **{'HTTP_HOST': 'localhost'}) - self.assertEqual(response.status_code, 400, response.content) - self.assertIn('Could not parse amount as', response.content) + self.assertEqual(response.status_code, 400, response.content.decode('utf-8')) + self.assertIn('Could not parse amount as', response.content.decode('utf-8')) def test_get_historical_coupon_codes(self): """ @@ -5228,11 +5237,11 @@ class TestCourseRegistrationCodes(SharedModuleStoreTestCase): code_redeemed_count="0", total_discounted_seats="0", total_discounted_amount="0", - ), response.content + ), response.content.decode("utf-8") ) self.assertEqual(response['Content-Type'], 'text/csv') - body = response.content.replace('\r', '') + body = response.content.decode('utf-8').replace('\r', '') self.assertTrue(body.startswith(EXPECTED_COUPON_CSV_HEADER)) diff --git a/lms/djangoapps/instructor/views/api.py b/lms/djangoapps/instructor/views/api.py index 5bb1cc91ba..6d5cd38f25 100644 --- a/lms/djangoapps/instructor/views/api.py +++ b/lms/djangoapps/instructor/views/api.py @@ -389,7 +389,7 @@ def register_and_enroll_students(request, course_id): # pylint: disable=too-man try: upload_file = request.FILES.get('students_list') if upload_file.name.endswith('.csv'): - students = [row for row in csv.reader(upload_file.read().splitlines())] + students = [row for row in csv.reader(upload_file.read().decode('utf-8').splitlines())] course = get_course_by_id(course_id) else: general_errors.append({ @@ -1405,7 +1405,11 @@ def _cohorts_csv_validator(file_storage, file_to_validate): Verifies that the expected columns are present in the CSV used to add users to cohorts. """ with file_storage.open(file_to_validate) as f: - reader = unicodecsv.reader(UniversalNewlineIterator(f), encoding='utf-8') + if six.PY2: + reader = unicodecsv.reader(UniversalNewlineIterator(f), encoding='utf-8') + else: + reader = csv.reader(f.read().decode('utf-8').splitlines()) + try: fieldnames = next(reader) except StopIteration: @@ -1954,10 +1958,10 @@ def get_anon_ids(request, course_id): # pylint: disable=unused-argument writer = csv.writer(response, dialect='excel', quotechar='"', quoting=csv.QUOTE_ALL) # In practice, there should not be non-ascii data in this query, # but trying to do the right thing anyway. - encoded = [text_type(s).encode('utf-8') for s in header] + encoded = [text_type(s) for s in header] writer.writerow(encoded) for row in rows: - encoded = [text_type(s).encode('utf-8') for s in row] + encoded = [text_type(s) for s in row] writer.writerow(encoded) return response