Fixing encode/decode issues for python3.

lms/djangoapps/verify_student/models.py

@@ -10,6 +10,7 @@ photo verification process as generic as possible.
 """
 from __future__ import absolute_import
 
+import codecs
 import functools
 import json
 import logging
@@ -620,7 +621,11 @@ class SoftwareSecurePhotoVerification(PhotoVerification):
             return
 
         aes_key_str = settings.VERIFY_STUDENT["SOFTWARE_SECURE"]["FACE_IMAGE_AES_KEY"]
-        aes_key = aes_key_str.decode("hex")
+
+        if six.PY3:
+            aes_key = codecs.decode(aes_key_str, "hex")
+        else:
+            aes_key = aes_key_str.decode("hex")
 
         path = self._get_path("face")
         buff = ContentFile(encrypt_and_encode(img_data, aes_key))
@@ -658,7 +663,11 @@ class SoftwareSecurePhotoVerification(PhotoVerification):
         self._storage.save(path, buff)
 
         # Update our record fields
-        self.photo_id_key = rsa_encrypted_aes_key.encode('base64')
+        if six.PY3:
+            self.photo_id_key = codecs.encode(rsa_encrypted_aes_key, 'base64')
+        else:
+            self.photo_id_key = rsa_encrypted_aes_key.encode('base64')
+
         self.save()
 
     @status_before_must_be("must_retry", "ready", "submitted")

lms/djangoapps/verify_student/ssencrypt.py

@@ -24,6 +24,7 @@ import logging
 import os
 from hashlib import md5, sha256
 
+import six
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.asymmetric.padding import MGF1, OAEP
@@ -83,7 +84,10 @@ def generate_aes_iv(key):
     Return the initialization vector Software Secure expects for a given AES
     key (they hash it a couple of times and take a substring).
     """
-    return md5(key + md5(key).hexdigest()).hexdigest()[:AES_BLOCK_SIZE_BYTES]
+    if six.PY3:
+        return md5(key + md5(key).hexdigest().encode('utf-8')).hexdigest()[:AES_BLOCK_SIZE_BYTES].encode('utf-8')
+    else:
+        return md5(key + md5(key).hexdigest()).hexdigest()[:AES_BLOCK_SIZE_BYTES]
 
 
 def random_aes_key():
@@ -92,6 +96,10 @@ def random_aes_key():
 
 def pad(data):
     """ Pad the given `data` such that it fits into the proper AES block size """
+
+    if six.PY3:
+        data = six.b(data)
+
     padder = PKCS7(AES.block_size).padder()
     return padder.update(data) + padder.finalize()