From bceba32f0a6410be9828a8cf3e88a8a87fe60402 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Mon, 25 Sep 2023 15:16:59 +0500 Subject: [PATCH] Upgrade django-storages to 1.14 ( max ver ) (#33312) * feat!: upgrading `django-storages` to `1.14` --- .../contentstore/tests/test_video_utils.py | 38 +++++++++---------- requirements/constraints.txt | 9 ++--- requirements/edx/base.txt | 23 +++++------ requirements/edx/development.txt | 23 +++++------ requirements/edx/doc.txt | 23 +++++------ requirements/edx/paver.txt | 2 +- requirements/edx/testing.txt | 24 +++++------- 7 files changed, 62 insertions(+), 80 deletions(-) diff --git a/cms/djangoapps/contentstore/tests/test_video_utils.py b/cms/djangoapps/contentstore/tests/test_video_utils.py index 80d55b92b8..c81761a283 100644 --- a/cms/djangoapps/contentstore/tests/test_video_utils.py +++ b/cms/djangoapps/contentstore/tests/test_video_utils.py @@ -5,7 +5,7 @@ Unit tests for video utils. from datetime import datetime from unittest import TestCase -from unittest.mock import MagicMock, patch +from unittest import mock import ddt import pytz @@ -144,7 +144,7 @@ class ScrapeVideoThumbnailsTestCase(CourseTestCase): return mocked_response @override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret') - @patch('requests.get') + @mock.patch('requests.get') @ddt.data( ( { @@ -228,7 +228,7 @@ class ScrapeVideoThumbnailsTestCase(CourseTestCase): self.assertEqual(thumbnail_content_type, 'image/jpeg') @override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret') - @patch('requests.get') + @mock.patch('requests.get') def test_scrape_youtube_thumbnail(self, mocked_request): """ Test that youtube thumbnails are correctly scrapped. @@ -273,8 +273,8 @@ class ScrapeVideoThumbnailsTestCase(CourseTestCase): ) ) @override_settings(AWS_ACCESS_KEY_ID='test_key_id', AWS_SECRET_ACCESS_KEY='test_secret') - @patch('cms.djangoapps.contentstore.video_utils.LOGGER') - @patch('requests.get') + @mock.patch('cms.djangoapps.contentstore.video_utils.LOGGER') + @mock.patch('requests.get') @ddt.unpack def test_scrape_youtube_thumbnail_logging( self, @@ -333,8 +333,8 @@ class ScrapeVideoThumbnailsTestCase(CourseTestCase): ) ), ) - @patch('cms.djangoapps.contentstore.video_utils.LOGGER') - @patch('cms.djangoapps.contentstore.video_utils.download_youtube_video_thumbnail') + @mock.patch('cms.djangoapps.contentstore.video_utils.LOGGER') + @mock.patch('cms.djangoapps.contentstore.video_utils.download_youtube_video_thumbnail') @ddt.unpack def test_no_video_thumbnail_downloaded( self, @@ -376,7 +376,7 @@ class S3Boto3TestCase(TestCase): def setUp(self): self.storage = S3Boto3Storage() - self.storage._connections.connection = MagicMock() # pylint: disable=protected-access + self.storage._connections.connection = mock.MagicMock() # pylint: disable=protected-access def order_dict(self, dictionary): """ @@ -417,18 +417,18 @@ class S3Boto3TestCase(TestCase): content = ContentFile('new content') storage = S3Boto3Storage(**{'bucket_name': 'test'}) - storage._connections.connection = MagicMock() # pylint: disable=protected-access + storage._connections.connection = mock.MagicMock() # pylint: disable=protected-access storage.save(name, content) storage.bucket.Object.assert_called_once_with(name) obj = storage.bucket.Object.return_value obj.upload_fileobj.assert_called_with( - content, + mock.ANY, ExtraArgs=self.order_dict({ 'ContentType': 'text/plain', }), - Config=storage._transfer_config # pylint: disable=protected-access + Config=storage.transfer_config # pylint: disable=protected-access ) @override_settings(AWS_DEFAULT_ACL='public-read') @@ -445,7 +445,7 @@ class S3Boto3TestCase(TestCase): name = 'test_storage_save.txt' content = ContentFile('new content') storage = S3Boto3Storage(**{'bucket_name': 'test', 'default_acl': default_acl}) - storage._connections.connection = MagicMock() # pylint: disable=protected-access + storage._connections.connection = mock.MagicMock() # pylint: disable=protected-access storage.save(name, content) storage.bucket.Object.assert_called_once_with(name) @@ -461,9 +461,9 @@ class S3Boto3TestCase(TestCase): del ExtraArgs['ACL'] obj.upload_fileobj.assert_called_with( - content, + mock.ANY, ExtraArgs=self.order_dict(ExtraArgs), - Config=storage._transfer_config # pylint: disable=protected-access + Config=storage.transfer_config # pylint: disable=protected-access ) @ddt.data('public-read', 'private') @@ -476,16 +476,16 @@ class S3Boto3TestCase(TestCase): content = ContentFile('new content') storage = S3Boto3Storage(**{'bucket_name': 'test', 'default_acl': None}) - storage._connections.connection = MagicMock() # pylint: disable=protected-access + storage._connections.connection = mock.MagicMock() # pylint: disable=protected-access storage.save(name, content) storage.bucket.Object.assert_called_once_with(name) obj = storage.bucket.Object.return_value obj.upload_fileobj.assert_called_with( - content, - ExtraArgs=self.order_dict({ + mock.ANY, + Config=storage.transfer_config, # pylint: disable=protected-access + ExtraArgs={ 'ContentType': 'text/plain', - }), - Config=storage._transfer_config # pylint: disable=protected-access + }, ) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 87fadf16e2..50c78c4851 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -20,9 +20,8 @@ celery>=5.2.2,<6.0.0 # required for celery>=5.2.0;<5.3.0 click>=8.0,<9.0 -# django-storages version upgrade -django-storages==1.13.2 - +# each version upgrade need release notes review. +django-storages==1.14 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better @@ -78,8 +77,7 @@ pylint<2.16.0 # greater version failing quality test. Fix them in seperate ticke # Deprecated version of the AWS SDK; # we should stop using this boto==2.39.0 -boto3==1.7.0 # Amazon Web Services SDK for Python -botocore==1.10.84 # via boto3, s3transfer + # adding these constraints to minimize boto3 and botocore changeset social-auth-core==4.3.0 @@ -134,3 +132,4 @@ openedx-learning==0.1.6 # existing custom parameter configurations unusable. # https://github.com/openedx/xblock-lti-consumer/issues/410 has been opened to track a fix lti-consumer-xblock==9.6.1 + diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 63a5befdd5..10891a48df 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -77,16 +77,14 @@ boto==2.39.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in -boto3==1.7.0 +boto3==1.28.53 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in # django-ses # fs-s3fs # ora2 -botocore==1.10.84 +botocore==1.31.53 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in # boto3 # s3transfer @@ -259,7 +257,7 @@ django-celery-results==2.5.1 # via -r requirements/edx/kernel.in django-classy-tags==4.1.0 # via django-sekizai -django-config-models==2.5.0 +django-config-models==2.5.1 # via # -r requirements/edx/kernel.in # edx-enterprise @@ -357,7 +355,7 @@ django-statici18n==2.4.0 # -r requirements/edx/kernel.in # lti-consumer-xblock # xblock-drag-and-drop-v2 -django-storages==1.13.2 +django-storages==1.14 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in @@ -402,10 +400,6 @@ djangorestframework==3.14.0 # super-csv djangorestframework-xml==2.0.0 # via edx-enterprise -docutils==0.19 - # via - # -c requirements/edx/../constraints.txt - # botocore done-xblock==2.1.0 # via -r requirements/edx/bundled.in drf-jwt==1.19.2 @@ -490,7 +484,7 @@ edx-enterprise==4.3.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in -edx-event-bus-kafka==5.4.0 +edx-event-bus-kafka==5.5.0 # via -r requirements/edx/kernel.in edx-event-bus-redis==0.3.1 # via -r requirements/edx/kernel.in @@ -500,7 +494,7 @@ edx-milestones==0.5.0 # via -r requirements/edx/kernel.in edx-name-affirmation==2.3.6 # via -r requirements/edx/kernel.in -edx-opaque-keys[django]==2.5.0 +edx-opaque-keys[django]==2.5.1 # via # -r requirements/edx/kernel.in # -r requirements/edx/paver.txt @@ -636,7 +630,7 @@ jinja2==3.1.2 # via # code-annotations # coreschema -jmespath==0.10.0 +jmespath==1.0.1 # via # boto3 # botocore @@ -1034,7 +1028,7 @@ rules==3.3 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.1.13 +s3transfer==0.6.2 # via boto3 sailthru-client==2.2.3 # via edx-ace @@ -1169,6 +1163,7 @@ urllib3==1.26.16 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/paver.txt + # botocore # elasticsearch # py2neo # requests diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 7c5173ccaa..28e9b4f9f1 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -145,17 +145,15 @@ boto==2.39.0 # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -boto3==1.7.0 +boto3==1.28.53 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # django-ses # fs-s3fs # ora2 -botocore==1.10.84 +botocore==1.31.53 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # boto3 @@ -431,7 +429,7 @@ django-classy-tags==4.1.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # django-sekizai -django-config-models==2.5.0 +django-config-models==2.5.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -571,7 +569,7 @@ django-statici18n==2.4.0 # -r requirements/edx/testing.txt # lti-consumer-xblock # xblock-drag-and-drop-v2 -django-storages==1.13.2 +django-storages==1.14 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt @@ -640,8 +638,6 @@ docutils==0.19 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt - # botocore # pydata-sphinx-theme # sphinx # sphinx-mdinclude @@ -759,7 +755,7 @@ edx-enterprise==4.3.1 # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -edx-event-bus-kafka==5.4.0 +edx-event-bus-kafka==5.5.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -782,7 +778,7 @@ edx-name-affirmation==2.3.6 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -edx-opaque-keys[django]==2.5.0 +edx-opaque-keys[django]==2.5.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -950,7 +946,7 @@ gitdb==4.0.10 # via # -r requirements/edx/doc.txt # gitpython -gitpython==3.1.36 +gitpython==3.1.37 # via -r requirements/edx/doc.txt glob2==0.7 # via @@ -1064,7 +1060,7 @@ jinja2==3.1.2 # coreschema # diff-cover # sphinx -jmespath==0.10.0 +jmespath==1.0.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1792,7 +1788,7 @@ rules==3.3 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.1.13 +s3transfer==0.6.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2117,6 +2113,7 @@ urllib3==1.26.16 # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt + # botocore # elasticsearch # pact-python # py2neo diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index d9cbb521ef..c168388afa 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -103,16 +103,14 @@ boto==2.39.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt -boto3==1.7.0 +boto3==1.28.53 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 -botocore==1.10.84 +botocore==1.31.53 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # boto3 # s3transfer @@ -312,7 +310,7 @@ django-classy-tags==4.1.0 # via # -r requirements/edx/base.txt # django-sekizai -django-config-models==2.5.0 +django-config-models==2.5.1 # via # -r requirements/edx/base.txt # edx-enterprise @@ -420,7 +418,7 @@ django-statici18n==2.4.0 # -r requirements/edx/base.txt # lti-consumer-xblock # xblock-drag-and-drop-v2 -django-storages==1.13.2 +django-storages==1.14 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt @@ -470,8 +468,6 @@ djangorestframework-xml==2.0.0 docutils==0.19 # via # -c requirements/edx/../constraints.txt - # -r requirements/edx/base.txt - # botocore # pydata-sphinx-theme # sphinx # sphinx-mdinclude @@ -564,7 +560,7 @@ edx-enterprise==4.3.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt -edx-event-bus-kafka==5.4.0 +edx-event-bus-kafka==5.5.0 # via -r requirements/edx/base.txt edx-event-bus-redis==0.3.1 # via -r requirements/edx/base.txt @@ -576,7 +572,7 @@ edx-milestones==0.5.0 # via -r requirements/edx/base.txt edx-name-affirmation==2.3.6 # via -r requirements/edx/base.txt -edx-opaque-keys[django]==2.5.0 +edx-opaque-keys[django]==2.5.1 # via # -r requirements/edx/base.txt # edx-bulk-grades @@ -686,7 +682,7 @@ geoip2==4.7.0 # via -r requirements/edx/base.txt gitdb==4.0.10 # via gitpython -gitpython==3.1.36 +gitpython==3.1.37 # via -r requirements/edx/doc.in glob2==0.7 # via -r requirements/edx/base.txt @@ -744,7 +740,7 @@ jinja2==3.1.2 # code-annotations # coreschema # sphinx -jmespath==0.10.0 +jmespath==1.0.1 # via # -r requirements/edx/base.txt # boto3 @@ -1222,7 +1218,7 @@ rules==3.3 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.1.13 +s3transfer==0.6.2 # via # -r requirements/edx/base.txt # boto3 @@ -1424,6 +1420,7 @@ urllib3==1.26.16 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt + # botocore # elasticsearch # py2neo # requests diff --git a/requirements/edx/paver.txt b/requirements/edx/paver.txt index 962fdec088..46af4d76cc 100644 --- a/requirements/edx/paver.txt +++ b/requirements/edx/paver.txt @@ -10,7 +10,7 @@ charset-normalizer==2.0.12 # via # -c requirements/edx/../constraints.txt # requests -edx-opaque-keys==2.5.0 +edx-opaque-keys==2.5.1 # via -r requirements/edx/paver.in idna==3.4 # via requests diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 2a07c19de0..6dc45b85a7 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -110,16 +110,14 @@ boto==2.39.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt -boto3==1.7.0 +boto3==1.28.53 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 -botocore==1.10.84 +botocore==1.31.53 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # boto3 # s3transfer @@ -345,7 +343,7 @@ django-classy-tags==4.1.0 # via # -r requirements/edx/base.txt # django-sekizai -django-config-models==2.5.0 +django-config-models==2.5.1 # via # -r requirements/edx/base.txt # edx-enterprise @@ -453,7 +451,7 @@ django-statici18n==2.4.0 # -r requirements/edx/base.txt # lti-consumer-xblock # xblock-drag-and-drop-v2 -django-storages==1.13.2 +django-storages==1.14 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt @@ -500,11 +498,6 @@ djangorestframework-xml==2.0.0 # via # -r requirements/edx/base.txt # edx-enterprise -docutils==0.19 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/base.txt - # botocore done-xblock==2.1.0 # via -r requirements/edx/base.txt drf-jwt==1.19.2 @@ -594,7 +587,7 @@ edx-enterprise==4.3.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt -edx-event-bus-kafka==5.4.0 +edx-event-bus-kafka==5.5.0 # via -r requirements/edx/base.txt edx-event-bus-redis==0.3.1 # via -r requirements/edx/base.txt @@ -609,7 +602,7 @@ edx-milestones==0.5.0 # via -r requirements/edx/base.txt edx-name-affirmation==2.3.6 # via -r requirements/edx/base.txt -edx-opaque-keys[django]==2.5.0 +edx-opaque-keys[django]==2.5.1 # via # -r requirements/edx/base.txt # edx-bulk-grades @@ -810,7 +803,7 @@ jinja2==3.1.2 # code-annotations # coreschema # diff-cover -jmespath==0.10.0 +jmespath==1.0.1 # via # -r requirements/edx/base.txt # boto3 @@ -1359,7 +1352,7 @@ rules==3.3 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.1.13 +s3transfer==0.6.2 # via # -r requirements/edx/base.txt # boto3 @@ -1564,6 +1557,7 @@ urllib3==1.26.16 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt + # botocore # elasticsearch # pact-python # py2neo