From 369885002d3b6a44c65a9dbc017127c152957f23 Mon Sep 17 00:00:00 2001 From: Usama Sadiq Date: Tue, 16 Mar 2021 15:15:35 +0500 Subject: [PATCH] refactor: ran pyupgrade on openedx apps (#26846) Ran pyupgrade on openedx/core/djangoapps/{coursegraph, courseware_api, crawlers} --- .../management/commands/dump_to_neo4j.py | 11 +++-- .../commands/tests/test_dump_to_neo4j.py | 30 +++++++------- .../management/commands/tests/utils.py | 10 ++--- openedx/core/djangoapps/coursegraph/tasks.py | 40 +++++++++---------- .../djangoapps/courseware_api/serializers.py | 2 +- .../courseware_api/tests/test_views.py | 12 +++--- .../core/djangoapps/courseware_api/urls.py | 8 ++-- .../core/djangoapps/courseware_api/views.py | 2 +- .../crawlers/migrations/0001_initial.py | 5 +-- .../migrations/0002_auto_20170419_0018.py | 3 -- openedx/core/djangoapps/crawlers/models.py | 11 +++-- .../djangoapps/crawlers/tests/test_models.py | 7 ++-- 12 files changed, 65 insertions(+), 76 deletions(-) diff --git a/openedx/core/djangoapps/coursegraph/management/commands/dump_to_neo4j.py b/openedx/core/djangoapps/coursegraph/management/commands/dump_to_neo4j.py index 742ec71e4f..ced9b7ac93 100644 --- a/openedx/core/djangoapps/coursegraph/management/commands/dump_to_neo4j.py +++ b/openedx/core/djangoapps/coursegraph/management/commands/dump_to_neo4j.py @@ -8,7 +8,6 @@ import logging from textwrap import dedent from django.core.management.base import BaseCommand -from django.utils import six from openedx.core.djangoapps.coursegraph.tasks import ModuleStoreSerializer @@ -39,14 +38,14 @@ class Command(BaseCommand): help = dedent(__doc__).strip() def add_arguments(self, parser): - parser.add_argument('--host', type=six.text_type) + parser.add_argument('--host', type=str) parser.add_argument('--https_port', type=int, default=7473) parser.add_argument('--http_port', type=int, default=7474) parser.add_argument('--secure', action='store_true') - parser.add_argument('--user', type=six.text_type) - parser.add_argument('--password', type=six.text_type) - parser.add_argument('--courses', type=six.text_type, nargs='*') - parser.add_argument('--skip', type=six.text_type, nargs='*') + parser.add_argument('--user', type=str) + parser.add_argument('--password', type=str) + parser.add_argument('--courses', type=str, nargs='*') + parser.add_argument('--skip', type=str, nargs='*') parser.add_argument( '--override', action='store_true', diff --git a/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py b/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py index f3544d504b..dac1b95089 100644 --- a/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py +++ b/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py @@ -1,4 +1,3 @@ -# coding=utf-8 """ Tests for the dump_to_neo4j management command. """ @@ -6,9 +5,8 @@ Tests for the dump_to_neo4j management command. from datetime import datetime +from unittest import mock import ddt -import mock -import six from django.core.management import call_command from edx_toggles.toggles.testutils import override_waffle_switch from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase @@ -59,7 +57,7 @@ class TestDumpToNeo4jCommandBase(SharedModuleStoreTestCase): test for a bug where xblocks with the same block_ids (but different locations) pointed to themselves erroneously. """ - super(TestDumpToNeo4jCommandBase, cls).setUpClass() + super().setUpClass() cls.course = CourseFactory.create() cls.chapter = ItemFactory.create(parent=cls.course, category='chapter') cls.sequential = ItemFactory.create(parent=cls.chapter, category='sequential') @@ -71,7 +69,7 @@ class TestDumpToNeo4jCommandBase(SharedModuleStoreTestCase): cls.course2 = CourseFactory.create() - cls.course_strings = [six.text_type(cls.course.id), six.text_type(cls.course2.id)] + cls.course_strings = [str(cls.course.id), str(cls.course2.id)] @staticmethod def setup_mock_graph(mock_selector_class, mock_graph_class, transaction_errors=False): @@ -105,7 +103,7 @@ class TestDumpToNeo4jCommandBase(SharedModuleStoreTestCase): number_commits: number of commits we expect against the graph number_rollbacks: number of commit rollbacks we expect """ - courses = set([node['course_key'] for node in mock_graph.nodes]) # lint-amnesty, pylint: disable=consider-using-set-comprehension + courses = {node['course_key'] for node in mock_graph.nodes} assert len(courses) == number_of_courses assert mock_graph.number_commits == number_commits assert mock_graph.number_rollbacks == number_rollbacks @@ -222,7 +220,7 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): ) -class SomeThing(object): +class SomeThing: """Just to test the stringification of an object.""" def __str__(self): return "" @@ -237,7 +235,7 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): @classmethod def setUpClass(cls): """Any ModuleStore course/content operations can go here.""" - super(TestModuleStoreSerializer, cls).setUpClass() + super().setUpClass() cls.mss = ModuleStoreSerializer.create() def test_serialize_item(self): @@ -316,7 +314,7 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): Returns: A tuple of the string representations of those XBlocks' locations. """ - return (six.text_type(xblock1.location), six.text_type(xblock2.location)) + return (str(xblock1.location), str(xblock2.location)) def assertBlockPairIsRelationship(self, xblock1, xblock2, relationships, relationship_type): """ @@ -423,7 +421,7 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): # 2 nodes and no relationships from the second assert len(mock_graph.nodes) == 11 - six.assertCountEqual(self, submitted, self.course_strings) + self.assertCountEqual(submitted, self.course_strings) @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeSelector') @mock.patch('openedx.core.djangoapps.coursegraph.tasks.authenticate_and_create_graph') @@ -447,7 +445,7 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): number_rollbacks=2, ) - six.assertCountEqual(self, submitted, self.course_strings) + self.assertCountEqual(submitted, self.course_strings) @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeSelector') @mock.patch('openedx.core.djangoapps.coursegraph.tasks.authenticate_and_create_graph') @@ -506,15 +504,15 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): # make sure only the published course was dumped submitted, __ = self.mss.dump_courses_to_neo4j(credentials) assert len(submitted) == 1 - assert submitted[0] == six.text_type(self.course.id) + assert submitted[0] == str(self.course.id) @mock.patch('openedx.core.djangoapps.coursegraph.tasks.get_course_last_published') @mock.patch('openedx.core.djangoapps.coursegraph.tasks.get_command_last_run') @ddt.data( - (six.text_type(datetime(2016, 3, 30)), six.text_type(datetime(2016, 3, 31)), True), - (six.text_type(datetime(2016, 3, 31)), six.text_type(datetime(2016, 3, 30)), False), - (six.text_type(datetime(2016, 3, 31)), None, False), - (None, six.text_type(datetime(2016, 3, 30)), True), + (str(datetime(2016, 3, 30)), str(datetime(2016, 3, 31)), True), + (str(datetime(2016, 3, 31)), str(datetime(2016, 3, 30)), False), + (str(datetime(2016, 3, 31)), None, False), + (None, str(datetime(2016, 3, 30)), True), (None, None, True), ) @ddt.unpack diff --git a/openedx/core/djangoapps/coursegraph/management/commands/tests/utils.py b/openedx/core/djangoapps/coursegraph/management/commands/tests/utils.py index b9f331af3c..34f7c17a26 100644 --- a/openedx/core/djangoapps/coursegraph/management/commands/tests/utils.py +++ b/openedx/core/djangoapps/coursegraph/management/commands/tests/utils.py @@ -6,7 +6,7 @@ Utilities for testing the dump_to_neo4j management command from py2neo import Node -class MockGraph(object): +class MockGraph: """ A stubbed out version of py2neo's Graph object, used for testing. Args: @@ -27,7 +27,7 @@ class MockGraph(object): return MockTransaction(self) -class MockTransaction(object): +class MockTransaction: """ A stubbed out version of py2neo's Transaction object, used for testing. """ @@ -50,9 +50,9 @@ class MockTransaction(object): end = query.find("'") course_key = query[:end] - self.graph.nodes = set([ # lint-amnesty, pylint: disable=consider-using-set-comprehension + self.graph.nodes = { node for node in self.graph.nodes if node['course_key'] != course_key - ]) + } def create(self, element): """ @@ -84,7 +84,7 @@ class MockTransaction(object): self.graph.number_rollbacks += 1 -class MockNodeSelector(object): +class MockNodeSelector: """ Mocks out py2neo's NodeSelector class. Used to select a node from a graph. py2neo's NodeSelector expects a real graph object to run queries against, diff --git a/openedx/core/djangoapps/coursegraph/tasks.py b/openedx/core/djangoapps/coursegraph/tasks.py index 716de57916..c046a48a77 100644 --- a/openedx/core/djangoapps/coursegraph/tasks.py +++ b/openedx/core/djangoapps/coursegraph/tasks.py @@ -8,7 +8,7 @@ import logging from celery import shared_task from django.conf import settings # lint-amnesty, pylint: disable=unused-import -from django.utils import six, timezone +from django.utils import timezone from edx_django_utils.cache import RequestCache from edx_django_utils.monitoring import set_code_owner_attribute from opaque_keys.edx.keys import CourseKey @@ -24,7 +24,7 @@ celery_log = logging.getLogger('edx.celery.task') bolt_log = logging.getLogger('neo4j.bolt') # pylint: disable=invalid-name bolt_log.setLevel(logging.ERROR) -PRIMITIVE_NEO4J_TYPES = (integer, string, six.text_type, float, bool) +PRIMITIVE_NEO4J_TYPES = (integer, string, str, float, bool) def serialize_item(item): @@ -40,23 +40,23 @@ def serialize_item(item): from xmodule.modulestore.store_utilities import DETACHED_XBLOCK_TYPES # convert all fields to a dict and filter out parent and children field - fields = dict( - (field, field_value.read_from(item)) - for (field, field_value) in six.iteritems(item.fields) + fields = { + field: field_value.read_from(item) + for (field, field_value) in item.fields.items() if field not in ['parent', 'children'] - ) + } course_key = item.scope_ids.usage_id.course_key block_type = item.scope_ids.block_type # set or reset some defaults - fields['edited_on'] = six.text_type(getattr(item, 'edited_on', '')) + fields['edited_on'] = str(getattr(item, 'edited_on', '')) fields['display_name'] = item.display_name_with_default fields['org'] = course_key.org fields['course'] = course_key.course fields['run'] = course_key.run - fields['course_key'] = six.text_type(course_key) - fields['location'] = six.text_type(item.location) + fields['course_key'] = str(course_key) + fields['location'] = str(item.location) fields['block_type'] = block_type fields['detached'] = block_type in DETACHED_XBLOCK_TYPES @@ -66,7 +66,7 @@ def serialize_item(item): del fields['checklists'] # record the time this command was run - fields['time_last_dumped_to_neo4j'] = six.text_type(timezone.now()) + fields['time_last_dumped_to_neo4j'] = str(timezone.now()) return fields, block_type @@ -81,12 +81,12 @@ def coerce_types(value): """ coerced_value = value if isinstance(value, list): - coerced_value = [six.text_type(element) for element in coerced_value] + coerced_value = [str(element) for element in coerced_value] # if it's not one of the types that neo4j accepts, # just convert it to text elif not isinstance(value, PRIMITIVE_NEO4J_TYPES): - coerced_value = six.text_type(value) + coerced_value = str(value) return coerced_value @@ -114,7 +114,7 @@ def get_command_last_run(course_key, graph): selector = NodeSelector(graph) course_node = selector.select( "course", - course_key=six.text_type(course_key) + course_key=str(course_key) ).first() last_this_command_was_run = None @@ -144,7 +144,7 @@ def get_course_last_published(course_key): course_usage_key = store.make_course_usage_key(course_key) try: structure = BlockStructureModel.get(course_usage_key) - course_last_published_date = six.text_type(structure.modified) + course_last_published_date = str(structure.modified) except BlockStructureNotFound: course_last_published_date = None @@ -183,7 +183,7 @@ def serialize_course(course_id): for item in items: fields, block_type = serialize_item(item) - for field_name, value in six.iteritems(fields): + for field_name, value in fields.items(): fields[field_name] = coerce_types(value) node = Node(block_type, 'item', **fields) @@ -270,7 +270,7 @@ def dump_course_to_neo4j(course_key_string, credentials): graph = authenticate_and_create_graph(credentials) transaction = graph.begin() - course_string = six.text_type(course_key) + course_string = str(course_key) try: # first, delete existing course transaction.run( @@ -293,7 +293,7 @@ def dump_course_to_neo4j(course_key_string, credentials): transaction.rollback() -class ModuleStoreSerializer(object): +class ModuleStoreSerializer: """ Class with functionality to serialize a modulestore into subgraphs, one graph per course. @@ -361,13 +361,13 @@ class ModuleStoreSerializer(object): if not (override_cache or should_dump_course(course_key, graph)): log.info("skipping submitting %s, since it hasn't changed", course_key) - skipped_courses.append(six.text_type(course_key)) + skipped_courses.append(str(course_key)) continue dump_course_to_neo4j.apply_async( - args=[six.text_type(course_key), credentials], + args=[str(course_key), credentials], ) - submitted_courses.append(six.text_type(course_key)) + submitted_courses.append(str(course_key)) return submitted_courses, skipped_courses diff --git a/openedx/core/djangoapps/courseware_api/serializers.py b/openedx/core/djangoapps/courseware_api/serializers.py index 4aab7012c7..6f33a4bf27 100644 --- a/openedx/core/djangoapps/courseware_api/serializers.py +++ b/openedx/core/djangoapps/courseware_api/serializers.py @@ -14,7 +14,7 @@ class _MediaSerializer(serializers.Serializer): # pylint: disable=abstract-meth """ def __init__(self, uri_attribute, *args, **kwargs): - super(_MediaSerializer, self).__init__(*args, **kwargs) # lint-amnesty, pylint: disable=super-with-arguments + super().__init__(*args, **kwargs) self.uri_attribute = uri_attribute uri = serializers.SerializerMethodField(source='*') diff --git a/openedx/core/djangoapps/courseware_api/tests/test_views.py b/openedx/core/djangoapps/courseware_api/tests/test_views.py index 56a1213f60..9976a7d6d8 100644 --- a/openedx/core/djangoapps/courseware_api/tests/test_views.py +++ b/openedx/core/djangoapps/courseware_api/tests/test_views.py @@ -6,8 +6,8 @@ from datetime import datetime from urllib.parse import urlencode from typing import Optional +from unittest import mock import ddt -import mock from completion.test_utils import CompletionWaffleTestMixin, submit_completions_for_testing from django.conf import settings from django.contrib.auth import get_user_model @@ -65,7 +65,7 @@ class BaseCoursewareTests(SharedModuleStoreTestCase): cls.user = UserFactory( username='student', - email=u'user@example.com', + email='user@example.com', password='foo', is_staff=False ) @@ -76,7 +76,7 @@ class BaseCoursewareTests(SharedModuleStoreTestCase): is_staff=False ) CourseInstructorRole(cls.course.id).add_users(cls.instructor) - cls.url = '/api/courseware/course/{}'.format(cls.course.id) + cls.url = f'/api/courseware/course/{cls.course.id}' @classmethod def tearDownClass(cls): @@ -297,7 +297,7 @@ class SequenceApiTestViews(BaseCoursewareTests): @classmethod def setUpClass(cls): super().setUpClass() - cls.url = '/api/courseware/sequence/{}'.format(cls.sequence.location) + cls.url = f'/api/courseware/sequence/{cls.sequence.location}' @classmethod def tearDownClass(cls): @@ -318,7 +318,7 @@ class ResumeApiTestViews(BaseCoursewareTests, CompletionWaffleTestMixin): @classmethod def setUpClass(cls): super().setUpClass() - cls.url = '/api/courseware/resume/{}'.format(cls.course.id) + cls.url = f'/api/courseware/resume/{cls.course.id}' def test_resume_no_completion(self): response = self.client.get(self.url) @@ -345,7 +345,7 @@ class CelebrationApiTestViews(BaseCoursewareTests, MasqueradeMixin): @classmethod def setUpClass(cls): super().setUpClass() - cls.url = '/api/courseware/celebration/{}'.format(cls.course.id) + cls.url = f'/api/courseware/celebration/{cls.course.id}' def setUp(self): super().setUp() diff --git a/openedx/core/djangoapps/courseware_api/urls.py b/openedx/core/djangoapps/courseware_api/urls.py index 6175da19a1..09903aed67 100644 --- a/openedx/core/djangoapps/courseware_api/urls.py +++ b/openedx/core/djangoapps/courseware_api/urls.py @@ -9,16 +9,16 @@ from django.conf.urls import url from openedx.core.djangoapps.courseware_api import views urlpatterns = [ - url(r'^course/{}'.format(settings.COURSE_KEY_PATTERN), + url(fr'^course/{settings.COURSE_KEY_PATTERN}', views.CoursewareInformation.as_view(), name="courseware-api"), - url(r'^sequence/{}'.format(settings.USAGE_KEY_PATTERN), + url(fr'^sequence/{settings.USAGE_KEY_PATTERN}', views.SequenceMetadata.as_view(), name="sequence-api"), - url(r'^resume/{}'.format(settings.COURSE_KEY_PATTERN), + url(fr'^resume/{settings.COURSE_KEY_PATTERN}', views.Resume.as_view(), name="resume-api"), - url(r'^celebration/{}'.format(settings.COURSE_KEY_PATTERN), + url(fr'^celebration/{settings.COURSE_KEY_PATTERN}', views.Celebration.as_view(), name="celebration-api"), ] diff --git a/openedx/core/djangoapps/courseware_api/views.py b/openedx/core/djangoapps/courseware_api/views.py index 40ea30ac22..5db49a5c72 100644 --- a/openedx/core/djangoapps/courseware_api/views.py +++ b/openedx/core/djangoapps/courseware_api/views.py @@ -489,7 +489,7 @@ class SequenceMetadata(DeveloperErrorViewMixin, APIView): try: usage_key = UsageKey.from_string(usage_key_string) except InvalidKeyError: - raise NotFound("Invalid usage key: '{}'.".format(usage_key_string)) # lint-amnesty, pylint: disable=raise-missing-from + raise NotFound(f"Invalid usage key: '{usage_key_string}'.") # lint-amnesty, pylint: disable=raise-missing-from _, request.user = setup_masquerade( request, diff --git a/openedx/core/djangoapps/crawlers/migrations/0001_initial.py b/openedx/core/djangoapps/crawlers/migrations/0001_initial.py index d8f6e371ec..336b31e1a8 100644 --- a/openedx/core/djangoapps/crawlers/migrations/0001_initial.py +++ b/openedx/core/djangoapps/crawlers/migrations/0001_initial.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - - from django.db import migrations, models import django.db.models.deletion from django.conf import settings @@ -19,7 +16,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('change_date', models.DateTimeField(auto_now_add=True, verbose_name='Change date')), ('enabled', models.BooleanField(default=False, verbose_name='Enabled')), - ('known_user_agents', models.TextField(default=u'edX-downloader', help_text=u'A comma-separated list of known crawler user agents.', blank=True)), + ('known_user_agents', models.TextField(default='edX-downloader', help_text='A comma-separated list of known crawler user agents.', blank=True)), ('changed_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, editable=False, to=settings.AUTH_USER_MODEL, null=True, verbose_name='Changed by')), ], options={ diff --git a/openedx/core/djangoapps/crawlers/migrations/0002_auto_20170419_0018.py b/openedx/core/djangoapps/crawlers/migrations/0002_auto_20170419_0018.py index 197e40c2d6..30b50e1d82 100644 --- a/openedx/core/djangoapps/crawlers/migrations/0002_auto_20170419_0018.py +++ b/openedx/core/djangoapps/crawlers/migrations/0002_auto_20170419_0018.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - - from django.db import migrations, models diff --git a/openedx/core/djangoapps/crawlers/models.py b/openedx/core/djangoapps/crawlers/models.py index 66619cfe50..34254277a1 100644 --- a/openedx/core/djangoapps/crawlers/models.py +++ b/openedx/core/djangoapps/crawlers/models.py @@ -4,7 +4,6 @@ appropriately in other parts of the code. """ -import six from config_models.models import ConfigurationModel from django.db import models from django.utils.encoding import python_2_unicode_compatible @@ -17,17 +16,17 @@ class CrawlersConfig(ConfigurationModel): .. no_pii: """ - class Meta(object): + class Meta: app_label = "crawlers" known_user_agents = models.TextField( blank=True, - help_text=u"A comma-separated list of known crawler user agents.", - default=u'edX-downloader', + help_text="A comma-separated list of known crawler user agents.", + default='edX-downloader', ) def __str__(self): - return u'CrawlersConfig("{}")'.format(self.known_user_agents) + return f'CrawlersConfig("{self.known_user_agents}")' @classmethod def is_crawler(cls, request): @@ -53,7 +52,7 @@ class CrawlersConfig(ConfigurationModel): # value is an ISO-8859-1 encoded byte string in Python 2.7 (and in the HTTP spec), but # it will be a unicode str when we move to Python 3.x. This code should work under # either version. - if isinstance(req_user_agent, six.binary_type): + if isinstance(req_user_agent, bytes): crawler_agents = [crawler_agent.encode('iso-8859-1') for crawler_agent in crawler_agents] # We perform prefix matching of the crawler agent here so that we don't diff --git a/openedx/core/djangoapps/crawlers/tests/test_models.py b/openedx/core/djangoapps/crawlers/tests/test_models.py index 5daf0284c4..e7813faa46 100644 --- a/openedx/core/djangoapps/crawlers/tests/test_models.py +++ b/openedx/core/djangoapps/crawlers/tests/test_models.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Tests that the request came from a crawler or not. """ @@ -14,7 +13,7 @@ from ..models import CrawlersConfig class CrawlersConfigTest(TestCase): # lint-amnesty, pylint: disable=missing-class-docstring def setUp(self): - super(CrawlersConfigTest, self).setUp() # lint-amnesty, pylint: disable=super-with-arguments + super().setUp() CrawlersConfig(known_user_agents='edX-downloader,crawler_foo', enabled=True).save() @ddt.data( @@ -31,8 +30,8 @@ class CrawlersConfigTest(TestCase): # lint-amnesty, pylint: disable=missing-cla assert not CrawlersConfig.is_crawler(fake_request) @ddt.data( - u"edX-downloader", - "crawler_foo".encode("utf-8") + "edX-downloader", + b"crawler_foo" ) def test_req_user_agent_is_crawler(self, req_user_agent): """