From 220cab965fbe3a2e4cf031ddb707a8f2844cd8e4 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Thu, 2 Jan 2020 11:50:26 +0500 Subject: [PATCH] BOM-1126 Removing the moto from repo. --- common/test/utils.py | 20 ++++++----- .../instructor_task/tests/test_models.py | 30 +++-------------- .../tests/test_populate_expiry_date.py | 11 ++----- .../test_send_verification_expiry_email.py | 8 ++--- .../commands/tests/test_verify_student.py | 10 ++---- .../verify_student/tests/test_models.py | 10 ++---- .../verify_student/tests/test_services.py | 3 +- .../verify_student/tests/test_views.py | 25 +++++++------- pavelib/paver_tests/test_database.py | 33 +++++++++---------- 9 files changed, 52 insertions(+), 98 deletions(-) diff --git a/common/test/utils.py b/common/test/utils.py index e4fa013fc0..8b78ba98ed 100644 --- a/common/test/utils.py +++ b/common/test/utils.py @@ -7,7 +7,6 @@ import functools import sys from contextlib import contextmanager -import moto import pytest from django.dispatch import Signal from markupsafe import escape @@ -117,19 +116,22 @@ def skip_signal(signal, **kwargs): signal.connect(**kwargs) -class MockS3Mixin(object): +class MockS3BotoMixin(object): """ - TestCase mixin that stubs S3 using the moto library. Note that this will - activate httpretty, which will monkey patch socket. + TestCase mixin that mocks the S3BotoStorage save method and s3 connection. """ def setUp(self): - super(MockS3Mixin, self).setUp() - self._mock_s3 = moto.mock_s3_deprecated() - self._mock_s3.start() + super(MockS3BotoMixin, self).setUp() + self._mocked_connection = patch('boto.connect_s3', return_value=Mock()) + self.mocked_connection = self._mocked_connection.start() + + self.patcher = patch('storages.backends.s3boto.S3BotoStorage.save') + self.patcher.start() def tearDown(self): - self._mock_s3.stop() - super(MockS3Mixin, self).tearDown() + self._mocked_connection.stop() + self.patcher.stop() + super(MockS3BotoMixin, self).tearDown() class reprwrapper(object): diff --git a/lms/djangoapps/instructor_task/tests/test_models.py b/lms/djangoapps/instructor_task/tests/test_models.py index 770b234c6a..77ebf579b4 100644 --- a/lms/djangoapps/instructor_task/tests/test_models.py +++ b/lms/djangoapps/instructor_task/tests/test_models.py @@ -7,13 +7,11 @@ import copy import time from six import StringIO -import boto from django.conf import settings from django.test import SimpleTestCase, TestCase, override_settings -from mock import patch from opaque_keys.edx.locator import CourseLocator -from common.test.utils import MockS3Mixin +from common.test.utils import MockS3BotoMixin from lms.djangoapps.instructor_task.models import InstructorTask, ReportStore, TASK_INPUT_LENGTH from lms.djangoapps.instructor_task.tests.test_base import TestReportMixin @@ -84,25 +82,6 @@ class LocalFSReportStoreTestCase(ReportStoreTestMixin, TestReportMixin, SimpleTe return ReportStore.from_config(config_name='GRADES_DOWNLOAD') -@patch.dict(settings.GRADES_DOWNLOAD, { - 'STORAGE_TYPE': 's3', - # Strip the leading `/`, because boto doesn't want it - 'ROOT_PATH': settings.GRADES_DOWNLOAD['ROOT_PATH'].lstrip('/') -}) -class S3ReportStoreTestCase(MockS3Mixin, ReportStoreTestMixin, TestReportMixin, SimpleTestCase): - """ - Test the old S3ReportStore configuration. - """ - def create_report_store(self): - """ - Create and return a DjangoStorageReportStore using the old - S3ReportStore configuration. - """ - connection = boto.connect_s3() - connection.create_bucket(settings.GRADES_DOWNLOAD['BUCKET']) - return ReportStore.from_config(config_name='GRADES_DOWNLOAD') - - class DjangoStorageReportStoreLocalTestCase(ReportStoreTestMixin, TestReportMixin, SimpleTestCase): """ Test the DjangoStorageReportStore implementation using the local @@ -119,7 +98,7 @@ class DjangoStorageReportStoreLocalTestCase(ReportStoreTestMixin, TestReportMixi return ReportStore.from_config(config_name='GRADES_DOWNLOAD') -class DjangoStorageReportStoreS3TestCase(MockS3Mixin, ReportStoreTestMixin, TestReportMixin, SimpleTestCase): +class DjangoStorageReportStoreS3TestCase(MockS3BotoMixin, ReportStoreTestMixin, TestReportMixin, SimpleTestCase): """ Test the DjangoStorageReportStore implementation using S3 stubs. """ @@ -135,12 +114,11 @@ class DjangoStorageReportStoreS3TestCase(MockS3Mixin, ReportStoreTestMixin, Test 'location': settings.GRADES_DOWNLOAD['ROOT_PATH'], } with override_settings(GRADES_DOWNLOAD=test_settings): - connection = boto.connect_s3() - connection.create_bucket(settings.GRADES_DOWNLOAD['STORAGE_KWARGS']['bucket']) + self.mocked_connection.create_bucket(settings.GRADES_DOWNLOAD['STORAGE_KWARGS']['bucket']) return ReportStore.from_config(config_name='GRADES_DOWNLOAD') -class TestS3ReportStorage(MockS3Mixin, TestCase): +class TestS3ReportStorage(TestCase): """ Test the S3ReportStorage to make sure that configuration overrides from settings.FINANCIAL_REPORTS are used instead of default ones. diff --git a/lms/djangoapps/verify_student/management/commands/tests/test_populate_expiry_date.py b/lms/djangoapps/verify_student/management/commands/tests/test_populate_expiry_date.py index 4a381cfac5..8a3ec881f9 100644 --- a/lms/djangoapps/verify_student/management/commands/tests/test_populate_expiry_date.py +++ b/lms/djangoapps/verify_student/management/commands/tests/test_populate_expiry_date.py @@ -5,14 +5,13 @@ Tests for django admin command `populate_expiry_date` in the verify_student modu from datetime import timedelta -import boto from django.conf import settings from django.core.management import call_command from django.test import TestCase from mock import patch from testfixtures import LogCapture -from common.test.utils import MockS3Mixin +from common.test.utils import MockS3BotoMixin from lms.djangoapps.verify_student.models import SoftwareSecurePhotoVerification from lms.djangoapps.verify_student.tests.test_models import FAKE_SETTINGS, mock_software_secure_post from student.tests.factories import UserFactory @@ -22,15 +21,9 @@ LOGGER_NAME = 'lms.djangoapps.verify_student.management.commands.populate_expiry @patch.dict(settings.VERIFY_STUDENT, FAKE_SETTINGS) @patch('lms.djangoapps.verify_student.models.requests.post', new=mock_software_secure_post) -class TestPopulateExpiryDate(MockS3Mixin, TestCase): +class TestPopulateExpiryDate(MockS3BotoMixin, TestCase): """ Tests for django admin command `populate_expiry_date` in the verify_student module """ - def setUp(self): - """ Initial set up for tests """ - super(TestPopulateExpiryDate, self).setUp() - connection = boto.connect_s3() - connection.create_bucket(FAKE_SETTINGS['SOFTWARE_SECURE']['S3_BUCKET']) - def create_and_submit(self, user): """ Helper method that lets us create new SoftwareSecurePhotoVerifications """ attempt = SoftwareSecurePhotoVerification(user=user) diff --git a/lms/djangoapps/verify_student/management/commands/tests/test_send_verification_expiry_email.py b/lms/djangoapps/verify_student/management/commands/tests/test_send_verification_expiry_email.py index 85d810d9e5..bde25e958e 100644 --- a/lms/djangoapps/verify_student/management/commands/tests/test_send_verification_expiry_email.py +++ b/lms/djangoapps/verify_student/management/commands/tests/test_send_verification_expiry_email.py @@ -5,18 +5,18 @@ Tests for django admin command `send_verification_expiry_email` in the verify_st from datetime import timedelta -import boto from django.conf import settings from django.contrib.sites.models import Site from django.core import mail from django.core.management import call_command, CommandError +from django.test import TestCase from django.test.utils import override_settings from django.utils.timezone import now from mock import patch from student.tests.factories import UserFactory from testfixtures import LogCapture -from common.test.utils import MockS3Mixin +from common.test.utils import MockS3BotoMixin from lms.djangoapps.verify_student.models import SoftwareSecurePhotoVerification from lms.djangoapps.verify_student.tests.test_models import FAKE_SETTINGS, mock_software_secure_post @@ -25,14 +25,12 @@ LOGGER_NAME = 'lms.djangoapps.verify_student.management.commands.send_verificati @patch.dict(settings.VERIFY_STUDENT, FAKE_SETTINGS) @patch('lms.djangoapps.verify_student.models.requests.post', new=mock_software_secure_post) -class TestSendVerificationExpiryEmail(MockS3Mixin): +class TestSendVerificationExpiryEmail(MockS3BotoMixin, TestCase): """ Tests for django admin command `send_verification_expiry_email` in the verify_student module """ def setUp(self): """ Initial set up for tests """ super(TestSendVerificationExpiryEmail, self).setUp() - connection = boto.connect_s3() - connection.create_bucket(FAKE_SETTINGS['SOFTWARE_SECURE']['S3_BUCKET']) Site.objects.create(domain='edx.org', name='edx.org') self.resend_days = settings.VERIFICATION_EXPIRY_EMAIL['RESEND_DAYS'] self.days = settings.VERIFICATION_EXPIRY_EMAIL['DAYS_RANGE'] diff --git a/lms/djangoapps/verify_student/management/commands/tests/test_verify_student.py b/lms/djangoapps/verify_student/management/commands/tests/test_verify_student.py index 76ac4864d6..1d71839e20 100644 --- a/lms/djangoapps/verify_student/management/commands/tests/test_verify_student.py +++ b/lms/djangoapps/verify_student/management/commands/tests/test_verify_student.py @@ -13,7 +13,7 @@ from django.test import TestCase from mock import patch from testfixtures import LogCapture -from common.test.utils import MockS3Mixin +from common.test.utils import MockS3BotoMixin from lms.djangoapps.verify_student.models import SoftwareSecurePhotoVerification, SSPVerificationRetryConfig from lms.djangoapps.verify_student.tests.test_models import ( FAKE_SETTINGS, @@ -28,16 +28,10 @@ LOGGER_NAME = 'retry_photo_verification' # Lots of patching to stub in our own settings, and HTTP posting @patch.dict(settings.VERIFY_STUDENT, FAKE_SETTINGS) @patch('lms.djangoapps.verify_student.models.requests.post', new=mock_software_secure_post) -class TestVerifyStudentCommand(MockS3Mixin, TestCase): +class TestVerifyStudentCommand(MockS3BotoMixin, TestCase): """ Tests for django admin commands in the verify_student module """ - - def setUp(self): - super(TestVerifyStudentCommand, self).setUp() - connection = boto.connect_s3() - connection.create_bucket(FAKE_SETTINGS['SOFTWARE_SECURE']['S3_BUCKET']) - def create_and_submit(self, username): """ Helper method that lets us create new SoftwareSecurePhotoVerifications diff --git a/lms/djangoapps/verify_student/tests/test_models.py b/lms/djangoapps/verify_student/tests/test_models.py index b9f25ea85b..fe59f36f23 100644 --- a/lms/djangoapps/verify_student/tests/test_models.py +++ b/lms/djangoapps/verify_student/tests/test_models.py @@ -5,7 +5,6 @@ import base64 import simplejson as json from datetime import datetime, timedelta -import boto import ddt import mock import requests.exceptions @@ -19,7 +18,7 @@ from student.tests.factories import UserFactory from testfixtures import LogCapture from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase -from common.test.utils import MockS3Mixin +from common.test.utils import MockS3BotoMixin from lms.djangoapps.verify_student.models import ( SoftwareSecurePhotoVerification, SSOVerification, @@ -123,12 +122,7 @@ class TestVerification(TestCase): @patch.dict(settings.VERIFY_STUDENT, FAKE_SETTINGS) @patch('lms.djangoapps.verify_student.models.requests.post', new=mock_software_secure_post) @ddt.ddt -class TestPhotoVerification(TestVerification, MockS3Mixin, ModuleStoreTestCase): - - def setUp(self): - super(TestPhotoVerification, self).setUp() - connection = boto.connect_s3() - connection.create_bucket(FAKE_SETTINGS['SOFTWARE_SECURE']['S3_BUCKET']) +class TestPhotoVerification(TestVerification, MockS3BotoMixin, ModuleStoreTestCase): def test_state_transitions(self): """ diff --git a/lms/djangoapps/verify_student/tests/test_services.py b/lms/djangoapps/verify_student/tests/test_services.py index ebed4ac61d..2e0a881436 100644 --- a/lms/djangoapps/verify_student/tests/test_services.py +++ b/lms/djangoapps/verify_student/tests/test_services.py @@ -8,7 +8,6 @@ import ddt from django.conf import settings from mock import patch -from common.test.utils import MockS3Mixin from lms.djangoapps.verify_student.models import ManualVerification, SoftwareSecurePhotoVerification, SSOVerification from lms.djangoapps.verify_student.services import IDVerificationService from student.tests.factories import UserFactory @@ -22,7 +21,7 @@ FAKE_SETTINGS = { @patch.dict(settings.VERIFY_STUDENT, FAKE_SETTINGS) @ddt.ddt -class TestIDVerificationService(ModuleStoreTestCase, MockS3Mixin): +class TestIDVerificationService(ModuleStoreTestCase): """ Tests for IDVerificationService. """ diff --git a/lms/djangoapps/verify_student/tests/test_views.py b/lms/djangoapps/verify_student/tests/test_views.py index dd97dcc6ec..374ad19795 100644 --- a/lms/djangoapps/verify_student/tests/test_views.py +++ b/lms/djangoapps/verify_student/tests/test_views.py @@ -4,16 +4,13 @@ Tests of verify_student views. """ -import simplejson as json from datetime import timedelta from uuid import uuid4 -import boto import ddt import httpretty import mock -import moto -import requests +import simplejson as json import six import six.moves.urllib.error # pylint: disable=import-error import six.moves.urllib.parse # pylint: disable=import-error @@ -33,7 +30,7 @@ from opaque_keys.edx.locator import CourseLocator from six.moves import zip from waffle.testutils import override_switch -from common.test.utils import XssTestMixin +from common.test.utils import MockS3BotoMixin, XssTestMixin from course_modes.models import CourseMode from course_modes.tests.factories import CourseModeFactory from lms.djangoapps.commerce.models import CommerceConfiguration @@ -1415,7 +1412,7 @@ class TestCreateOrderView(ModuleStoreTestCase): @ddt.ddt @patch.dict(settings.FEATURES, {'AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING': True}) -class TestSubmitPhotosForVerification(TestCase): +class TestSubmitPhotosForVerification(MockS3BotoMixin, TestCase): """ Tests for submitting photos for verification. """ @@ -1491,20 +1488,20 @@ class TestSubmitPhotosForVerification(TestCase): }, "DAYS_GOOD_FOR": 10, }) - @moto.mock_s3_deprecated + @httpretty.activate def test_submit_photos_for_reverification(self): - # Create the S3 bucket for photo upload - conn = boto.connect_s3() - conn.create_bucket("test.example.com") - - # Mock the POST to Software Secure - moto.packages.httpretty.register_uri(httpretty.POST, "https://verify.example.com/submit/") + httpretty.register_uri( + httpretty.POST, settings.VERIFY_STUDENT["SOFTWARE_SECURE"]["API_URL"], + status=200, body={}, + content_type='application/json' + ) # Submit an initial verification attempt self._submit_photos( face_image=self.IMAGE_DATA + "4567", photo_id_image=self.IMAGE_DATA + "8910", ) + initial_data = self._get_post_data() # Submit a face photo for re-verification @@ -1635,7 +1632,7 @@ class TestSubmitPhotosForVerification(TestCase): def _get_post_data(self): """Retrieve POST data from the last request. """ - last_request = moto.packages.httpretty.last_request() + last_request = httpretty.last_request() return json.loads(last_request.body) diff --git a/pavelib/paver_tests/test_database.py b/pavelib/paver_tests/test_database.py index 128eda6c33..0c6e9b7436 100644 --- a/pavelib/paver_tests/test_database.py +++ b/pavelib/paver_tests/test_database.py @@ -10,12 +10,11 @@ from tempfile import mkdtemp from unittest import TestCase import boto -from mock import call, patch +from mock import call, patch, Mock -from common.test.utils import MockS3Mixin from pavelib import database from pavelib.utils import db_utils -from pavelib.utils.db_utils import extract_files_from_zip, is_fingerprint_in_bucket +from pavelib.utils.db_utils import extract_files_from_zip from pavelib.utils.envs import Env from .utils import PaverTestCase @@ -60,20 +59,18 @@ def _write_temporary_db_cache_files(path, files): cache_file.write(str(index)) -class TestPaverDatabaseTasks(MockS3Mixin, PaverTestCase): +class TestPaverDatabaseTasks(PaverTestCase): """ Tests for the high level database tasks """ def setUp(self): super(TestPaverDatabaseTasks, self).setUp() - conn = boto.connect_s3() - conn.create_bucket('moto_test_bucket', policy='public-read') - self.bucket = conn.get_bucket('moto_test_bucket') # This value is the actual sha1 fingerprint calculated for the dummy # files used in these tests self.expected_fingerprint = 'ccaa8d8dcc7d030cd6a6768db81f90d0ef976c3d' self.fingerprint_filename = '{}.tar.gz'.format(self.expected_fingerprint) + self.bucket = Mock(name='test_bucket') @patch.object(db_utils, 'CACHE_FOLDER', mkdtemp()) @patch.object(db_utils, 'FINGERPRINT_FILEPATH', os.path.join(mkdtemp(), 'fingerprint')) @@ -102,7 +99,7 @@ class TestPaverDatabaseTasks(MockS3Mixin, PaverTestCase): ] _mock_sh.assert_has_calls(calls) - @patch.object(database, 'CACHE_BUCKET_NAME', 'moto_test_bucket') + @patch.object(database, 'CACHE_BUCKET_NAME', 'test_bucket') @patch.object(db_utils, 'CACHE_FOLDER', mkdtemp()) @patch.object(db_utils, 'FINGERPRINT_FILEPATH', os.path.join(mkdtemp(), 'fingerprint')) @patch.object(db_utils, 'sh') @@ -117,7 +114,7 @@ class TestPaverDatabaseTasks(MockS3Mixin, PaverTestCase): self.addCleanup(os.remove, db_utils.FINGERPRINT_FILEPATH) _write_temporary_db_cache_files(db_utils.CACHE_FOLDER, database.ALL_DB_FILES) - # zip the temporary files and push them to a moto s3 bucket + # zip the temporary files and push them to s3 bucket zipfile_path = os.path.join(db_utils.CACHE_FOLDER, self.fingerprint_filename) with tarfile.open(name=zipfile_path, mode='w:gz') as tar_file: for name in database.ALL_DB_FILES: @@ -131,19 +128,21 @@ class TestPaverDatabaseTasks(MockS3Mixin, PaverTestCase): with open(db_utils.FINGERPRINT_FILEPATH, 'w') as fingerprint_file: fingerprint_file.write(local_fingerprint) - with patch.object(db_utils, 'get_file_from_s3', wraps=db_utils.get_file_from_s3) as _mock_get_file: - database.update_local_bokchoy_db_from_s3() # pylint: disable=no-value-for-parameter - # Make sure that the fingerprint file is downloaded from s3 - _mock_get_file.assert_called_once_with( - 'moto_test_bucket', self.fingerprint_filename, db_utils.CACHE_FOLDER - ) + with patch('boto.connect_s3', Mock(return_value=Mock())): + with patch.object(db_utils, 'get_file_from_s3') as _mock_get_file: + database.update_local_bokchoy_db_from_s3() # pylint: disable=no-value-for-parameter + # Make sure that the fingerprint file is downloaded from s3 + _mock_get_file.assert_called_once_with( + 'test_bucket', self.fingerprint_filename, db_utils.CACHE_FOLDER + ) + calls = [ call(u'{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)), call(u'{}/scripts/reset-test-db.sh --use-existing-db'.format(Env.REPO_ROOT)) ] _mock_sh.assert_has_calls(calls) - @patch.object(database, 'CACHE_BUCKET_NAME', 'moto_test_bucket') + @patch.object(database, 'CACHE_BUCKET_NAME', 'test_bucket') @patch.object(db_utils, 'CACHE_FOLDER', mkdtemp()) @patch.object(db_utils, 'FINGERPRINT_FILEPATH', os.path.join(mkdtemp(), 'fingerprint')) @patch.object(db_utils, 'sh') @@ -171,7 +170,7 @@ class TestPaverDatabaseTasks(MockS3Mixin, PaverTestCase): ] _mock_sh.assert_has_calls(calls) - @patch.object(database, 'CACHE_BUCKET_NAME', 'moto_test_bucket') + @patch.object(database, 'CACHE_BUCKET_NAME', 'test_bucket') @patch.object(db_utils, 'CACHE_FOLDER', mkdtemp()) @patch.object(db_utils, 'FINGERPRINT_FILEPATH', os.path.join(mkdtemp(), 'fingerprint')) @patch.object(db_utils, 'sh')