From cd3957b98723e9788b97396fb5d060ac49463065 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Wed, 25 Aug 2021 09:34:41 -0400 Subject: [PATCH] fix: upgrade py2neo from 3.1.2 to 2021.1.5 (#28480) * fix: upgrade py2neo from 3.1.2 to 2021.1.5 The dump_to_neo4j management command has not been working since the upgrade to python 3.8. The latest version of python that py2neo 3.1.2 states support for is python 3.5, so this isn't surprising. The earliest non-prerelease version of py2neo that supports python 3.8 is 2020.x (skipping the 4.x and 5.x series). Since we're going as far as a 2020.x, we may as well upgrade all the way to the newest series, 2021.x. This commit does that upgrade, as well as a handful of minor code modifications in order to handle breaking changes that have been made to the py2neo API, and some unrelated pin bumps as the result of 'make upgrade'. This will also require an upgrade of Coursegraph's Neo4j version from 3.2.x to 3.5.x. TNL-8386 --- .../management/commands/dump_to_neo4j.py | 8 +- .../commands/tests/test_dump_to_neo4j.py | 72 ++++++++-------- .../management/commands/tests/utils.py | 58 ++++++------- openedx/core/djangoapps/coursegraph/tasks.py | 31 +++---- requirements/constraints.txt | 6 ++ requirements/edx-sandbox/py35.txt | 1 - requirements/edx-sandbox/py38.txt | 2 +- requirements/edx/base.in | 1 + requirements/edx/base.txt | 49 ++++++++--- requirements/edx/development.txt | 85 +++++++++++++++++-- requirements/edx/doc.txt | 4 +- requirements/edx/github.in | 1 - requirements/edx/testing.txt | 74 ++++++++++++++-- 13 files changed, 268 insertions(+), 124 deletions(-) diff --git a/openedx/core/djangoapps/coursegraph/management/commands/dump_to_neo4j.py b/openedx/core/djangoapps/coursegraph/management/commands/dump_to_neo4j.py index ced9b7ac93..226b3c174a 100644 --- a/openedx/core/djangoapps/coursegraph/management/commands/dump_to_neo4j.py +++ b/openedx/core/djangoapps/coursegraph/management/commands/dump_to_neo4j.py @@ -20,9 +20,8 @@ class Command(BaseCommand): Takes the following named arguments: host: the host of the neo4j server - https_port: the port on the neo4j server that accepts https requests - http_port: the port on the neo4j server that accepts http requests - secure: if set, connects to server over https, otherwise uses http + port: the port on the neo4j server that accepts Bolt requests + secure: if set, connects to server over Bolt/TLS, otherwise uses Bolt user: the username for the neo4j user password: the user's password courses: list of course key strings to serialize. If not specified, all @@ -39,8 +38,7 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('--host', type=str) - parser.add_argument('--https_port', type=int, default=7473) - parser.add_argument('--http_port', type=int, default=7474) + parser.add_argument('--port', type=int, default=7687) parser.add_argument('--secure', action='store_true') parser.add_argument('--user', type=str) parser.add_argument('--password', type=str) diff --git a/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py b/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py index df47802fe8..3e5699a605 100644 --- a/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py +++ b/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py @@ -15,7 +15,7 @@ from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory import openedx.core.djangoapps.content.block_structure.config as block_structure_config from openedx.core.djangoapps.content.block_structure.signals import update_block_structure_on_course_publish from openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j import ModuleStoreSerializer -from openedx.core.djangoapps.coursegraph.management.commands.tests.utils import MockGraph, MockNodeSelector +from openedx.core.djangoapps.coursegraph.management.commands.tests.utils import MockGraph, MockNodeMatcher from openedx.core.djangoapps.coursegraph.tasks import ( coerce_types, serialize_course, @@ -72,13 +72,13 @@ class TestDumpToNeo4jCommandBase(SharedModuleStoreTestCase): cls.course_strings = [str(cls.course.id), str(cls.course2.id)] @staticmethod - def setup_mock_graph(mock_selector_class, mock_graph_class, transaction_errors=False): + def setup_mock_graph(mock_matcher_class, mock_graph_class, transaction_errors=False): """ Replaces the py2neo Graph object with a MockGraph; similarly replaces - NodeSelector with MockNodeSelector. + NodeMatcher with MockNodeMatcher. Arguments: - mock_selector_class: a mocked NodeSelector class + mock_matcher_class: a mocked NodeMatcher class mock_graph_class: a mocked Graph class transaction_errors: a bool for whether we should get errors when transactions try to commit @@ -89,8 +89,8 @@ class TestDumpToNeo4jCommandBase(SharedModuleStoreTestCase): mock_graph = MockGraph(transaction_errors=transaction_errors) mock_graph_class.return_value = mock_graph - mock_node_selector = MockNodeSelector(mock_graph) - mock_selector_class.return_value = mock_node_selector + mock_node_matcher = MockNodeMatcher(mock_graph) + mock_matcher_class.return_value = mock_node_matcher return mock_graph def assertCourseDump(self, mock_graph, number_of_courses, number_commits, number_rollbacks): @@ -115,20 +115,20 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): Tests for the dump to neo4j management command """ - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeSelector') + @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') @mock.patch('openedx.core.djangoapps.coursegraph.tasks.Graph') @ddt.data(1, 2) - def test_dump_specific_courses(self, number_of_courses, mock_graph_class, mock_selector_class): + def test_dump_specific_courses(self, number_of_courses, mock_graph_class, mock_matcher_class): """ Test that you can specify which courses you want to dump. """ - mock_graph = self.setup_mock_graph(mock_selector_class, mock_graph_class) + mock_graph = self.setup_mock_graph(mock_matcher_class, mock_graph_class) call_command( 'dump_to_neo4j', courses=self.course_strings[:number_of_courses], host='mock_host', - http_port=7474, + port=7687, user='mock_user', password='mock_password', ) @@ -140,21 +140,21 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): number_rollbacks=0 ) - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeSelector') + @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') @mock.patch('openedx.core.djangoapps.coursegraph.tasks.Graph') - def test_dump_skip_course(self, mock_graph_class, mock_selector_class): + def test_dump_skip_course(self, mock_graph_class, mock_matcher_class): """ Test that you can skip courses. """ mock_graph = self.setup_mock_graph( - mock_selector_class, mock_graph_class + mock_matcher_class, mock_graph_class ) call_command( 'dump_to_neo4j', skip=self.course_strings[:1], host='mock_host', - http_port=7474, + port=7687, user='mock_user', password='mock_password', ) @@ -166,14 +166,14 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): number_rollbacks=0, ) - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeSelector') + @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') @mock.patch('openedx.core.djangoapps.coursegraph.tasks.Graph') - def test_dump_skip_beats_specifying(self, mock_graph_class, mock_selector_class): + def test_dump_skip_beats_specifying(self, mock_graph_class, mock_matcher_class): """ Test that if you skip and specify the same course, you'll skip it. """ mock_graph = self.setup_mock_graph( - mock_selector_class, mock_graph_class + mock_matcher_class, mock_graph_class ) call_command( @@ -181,7 +181,7 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): skip=self.course_strings[:1], courses=self.course_strings[:1], host='mock_host', - http_port=7474, + port=7687, user='mock_user', password='mock_password', ) @@ -193,21 +193,21 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): number_rollbacks=0, ) - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeSelector') + @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') @mock.patch('openedx.core.djangoapps.coursegraph.tasks.Graph') - def test_dump_all_courses(self, mock_graph_class, mock_selector_class): + def test_dump_all_courses(self, mock_graph_class, mock_matcher_class): """ Test if you don't specify which courses to dump, then you'll dump all of them. """ mock_graph = self.setup_mock_graph( - mock_selector_class, mock_graph_class + mock_matcher_class, mock_graph_class ) call_command( 'dump_to_neo4j', host='mock_host', - http_port=7474, + port=7687, user='mock_user', password='mock_password' ) @@ -297,8 +297,8 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): constituent nodes. """ relationship_pairs = [ - tuple([node["location"] for node in rel.nodes()]) # lint-amnesty, pylint: disable=consider-using-generator - for rel in relationships if rel.type() == relationship_type + (rel.start_node["location"], rel.end_node["location"]) + for rel in relationships if type(rel).__name__ == relationship_type ] return relationship_pairs @@ -395,16 +395,16 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): coerced_value = coerce_types(original_value) assert coerced_value == coerced_expected - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeSelector') + @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') @mock.patch('openedx.core.djangoapps.coursegraph.tasks.authenticate_and_create_graph') - def test_dump_to_neo4j(self, mock_graph_constructor, mock_selector_class): + def test_dump_to_neo4j(self, mock_graph_constructor, mock_matcher_class): """ Tests the dump_to_neo4j method works against a mock py2neo Graph """ mock_graph = MockGraph() mock_graph_constructor.return_value = mock_graph - mock_selector_class.return_value = MockNodeSelector(mock_graph) + mock_matcher_class.return_value = MockNodeMatcher(mock_graph) # mocking is thorwing error in kombu serialzier and its not require here any more. credentials = {} @@ -423,16 +423,16 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): assert len(mock_graph.nodes) == 11 self.assertCountEqual(submitted, self.course_strings) - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeSelector') + @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') @mock.patch('openedx.core.djangoapps.coursegraph.tasks.authenticate_and_create_graph') - def test_dump_to_neo4j_rollback(self, mock_graph_constructor, mock_selector_class): + def test_dump_to_neo4j_rollback(self, mock_graph_constructor, mock_matcher_class): """ Tests that the the dump_to_neo4j method handles the case where there's an exception trying to write to the neo4j database. """ mock_graph = MockGraph(transaction_errors=True) mock_graph_constructor.return_value = mock_graph - mock_selector_class.return_value = MockNodeSelector(mock_graph) + mock_matcher_class.return_value = MockNodeMatcher(mock_graph) # mocking is thorwing error in kombu serialzier and its not require here any more. credentials = {} @@ -447,7 +447,7 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): self.assertCountEqual(submitted, self.course_strings) - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeSelector') + @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') @mock.patch('openedx.core.djangoapps.coursegraph.tasks.authenticate_and_create_graph') @ddt.data((True, 2), (False, 0)) @ddt.unpack @@ -456,7 +456,7 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): override_cache, expected_number_courses, mock_graph_constructor, - mock_selector_class, + mock_matcher_class, ): """ Tests the caching mechanism and override to make sure we only publish @@ -464,7 +464,7 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): """ mock_graph = MockGraph() mock_graph_constructor.return_value = mock_graph - mock_selector_class.return_value = MockNodeSelector(mock_graph) + mock_matcher_class.return_value = MockNodeMatcher(mock_graph) # mocking is thorwing error in kombu serialzier and its not require here any more. credentials = {} @@ -480,16 +480,16 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): ) assert len(submitted) == expected_number_courses - @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeSelector') + @mock.patch('openedx.core.djangoapps.coursegraph.tasks.NodeMatcher') @mock.patch('openedx.core.djangoapps.coursegraph.tasks.authenticate_and_create_graph') - def test_dump_to_neo4j_published(self, mock_graph_constructor, mock_selector_class): + def test_dump_to_neo4j_published(self, mock_graph_constructor, mock_matcher_class): """ Tests that we only dump those courses that have been published after the last time the command was been run. """ mock_graph = MockGraph() mock_graph_constructor.return_value = mock_graph - mock_selector_class.return_value = MockNodeSelector(mock_graph) + mock_matcher_class.return_value = MockNodeMatcher(mock_graph) # mocking is thorwing error in kombu serialzier and its not require here any more. credentials = {} diff --git a/openedx/core/djangoapps/coursegraph/management/commands/tests/utils.py b/openedx/core/djangoapps/coursegraph/management/commands/tests/utils.py index 34f7c17a26..c1b776b7bf 100644 --- a/openedx/core/djangoapps/coursegraph/management/commands/tests/utils.py +++ b/openedx/core/djangoapps/coursegraph/management/commands/tests/utils.py @@ -26,6 +26,26 @@ class MockGraph: """ return MockTransaction(self) + def commit(self, transaction): + """ + Takes elements in the mock transaction's temporary storage and adds them + to this mock graph's storage. Throws an error if this graph's + transaction_errors param is set to True. + """ + if self.transaction_errors: + raise Exception("fake exception while trying to commit") + for element in transaction.temp: + self.nodes.add(element) + transaction.temp.clear() + self.number_commits += 1 + + def rollback(self, transaction): + """ + Clears the transactions temporary storage + """ + transaction.temp.clear() + self.number_rollbacks += 1 + class MockTransaction: """ @@ -63,38 +83,18 @@ class MockTransaction: if isinstance(element, Node): self.temp.add(element) - def commit(self): - """ - Takes elements in the transaction's temporary storage and adds them - to the mock graph's storage. Throws an error if the graph's - transaction_errors param is set to True. - """ - if self.graph.transaction_errors: - raise Exception("fake exception while trying to commit") - for element in self.temp: - self.graph.nodes.add(element) - self.temp.clear() - self.graph.number_commits += 1 - def rollback(self): - """ - Clears the transactions temporary storage - """ - self.temp.clear() - self.graph.number_rollbacks += 1 - - -class MockNodeSelector: +class MockNodeMatcher: """ - Mocks out py2neo's NodeSelector class. Used to select a node from a graph. - py2neo's NodeSelector expects a real graph object to run queries against, + Mocks out py2neo's NodeMatcher class. Used to match a node from a graph. + py2neo's NodeMatcher expects a real graph object to run queries against, so, rather than have to mock out MockGraph to accommodate those queries, - it seemed simpler to mock out NodeSelector as well. + it seemed simpler to mock out NodeMatcher as well. """ def __init__(self, graph): self.graph = graph - def select(self, label, course_key): + def match(self, label, course_key): """ Selects nodes that match a label and course_key Args: @@ -107,13 +107,13 @@ class MockNodeSelector: for node in self.graph.nodes: if node.has_label(label) and node["course_key"] == course_key: nodes.append(node) - return MockNodeSelection(nodes) + return MockNodeMatch(nodes) -class MockNodeSelection(list): +class MockNodeMatch(list): """ - Mocks out py2neo's NodeSelection class: this is the type of what - MockNodeSelector's `select` method returns. + Mocks out py2neo's NodeMatch class: this is the type of what + MockNodeMatcher's `match` method returns. """ def first(self): """ diff --git a/openedx/core/djangoapps/coursegraph/tasks.py b/openedx/core/djangoapps/coursegraph/tasks.py index 29da568ce0..67c020afbf 100644 --- a/openedx/core/djangoapps/coursegraph/tasks.py +++ b/openedx/core/djangoapps/coursegraph/tasks.py @@ -11,8 +11,7 @@ from django.utils import timezone from edx_django_utils.cache import RequestCache from edx_django_utils.monitoring import set_code_owner_attribute from opaque_keys.edx.keys import CourseKey -from py2neo import Graph, Node, Relationship, authenticate, NodeSelector -from py2neo.compat import integer, string +from py2neo import Graph, Node, Relationship, NodeMatcher log = logging.getLogger(__name__) @@ -23,7 +22,7 @@ celery_log = logging.getLogger('edx.celery.task') bolt_log = logging.getLogger('neo4j.bolt') # pylint: disable=invalid-name bolt_log.setLevel(logging.ERROR) -PRIMITIVE_NEO4J_TYPES = (integer, string, str, float, bool) +PRIMITIVE_NEO4J_TYPES = (int, bytes, str, float, bool) def serialize_item(item): @@ -110,8 +109,8 @@ def get_command_last_run(course_key, graph): Returns: The datetime that the command was last run, converted into text, or None, if there's no record of this command last being run. """ - selector = NodeSelector(graph) - course_node = selector.select( + matcher = NodeMatcher(graph) + course_node = matcher.match( "course", course_key=str(course_key) ).first() @@ -281,7 +280,7 @@ def dump_course_to_neo4j(course_key_string, credentials): # now, re-add it add_to_transaction(nodes, transaction) add_to_transaction(relationships, transaction) - transaction.commit() + graph.commit(transaction) celery_log.info("Completed dumping %s to neo4j", course_key) except Exception: # pylint: disable=broad-except @@ -289,7 +288,7 @@ def dump_course_to_neo4j(course_key_string, credentials): "Error trying to dump course %s to neo4j, rolling back", course_string ) - transaction.rollback() + graph.rollback(transaction) class ModuleStoreSerializer: @@ -382,27 +381,17 @@ def authenticate_and_create_graph(credentials): """ host = credentials['host'] - https_port = credentials['https_port'] - http_port = credentials['http_port'] + port = credentials['port'] secure = credentials['secure'] neo4j_user = credentials['user'] neo4j_password = credentials['password'] - authenticate( - "{host}:{port}".format( - host=host, port=https_port if secure else http_port - ), - neo4j_user, - neo4j_password, - ) - graph = Graph( - bolt=True, + protocol='bolt', password=neo4j_password, user=neo4j_user, - https_port=https_port, - http_port=http_port, - host=host, + address=host, + port=port, secure=secure, ) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index d86aecef07..330b1fedfc 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -107,3 +107,9 @@ django-ipware<4.0.0 # pylint>=2.10.0 introduced a lot of new pylint warnings. # A separate PR will be needed to remove the pin and fix all the pylint warnings pylint<2.10.0 + +# At the time of writing this comment, we do not know whether py2neo>=2022 +# will support our currently-deployed Neo4j version (3.5). +# Feel free to loosen this constraint if/when it is confirmed that a later +# version of py2neo will work with Neo4j 3.5. +py2neo<2022 diff --git a/requirements/edx-sandbox/py35.txt b/requirements/edx-sandbox/py35.txt index be7b526ba7..d8ed570665 100644 --- a/requirements/edx-sandbox/py35.txt +++ b/requirements/edx-sandbox/py35.txt @@ -62,7 +62,6 @@ numpy==1.16.5 # chem # matplotlib # openedx-calc - # scipy openedx-calc==1.0.9 # via -r requirements/edx-sandbox/py35.in pycparser==2.20 diff --git a/requirements/edx-sandbox/py38.txt b/requirements/edx-sandbox/py38.txt index 021e92d9c6..d8557d48a8 100644 --- a/requirements/edx-sandbox/py38.txt +++ b/requirements/edx-sandbox/py38.txt @@ -16,7 +16,7 @@ click==7.1.2 # via # -c requirements/edx-sandbox/../constraints.txt # nltk -cryptography==3.4.7 +cryptography==3.4.8 # via -r requirements/edx-sandbox/py38.in cycler==0.10.0 # via matplotlib diff --git a/requirements/edx/base.in b/requirements/edx/base.in index 4706abbf0d..0bc7b01724 100644 --- a/requirements/edx/base.in +++ b/requirements/edx/base.in @@ -124,6 +124,7 @@ openedx-calc # Library supporting mathematical calculatio ora2 piexif # Exif image metadata manipulation, used in the profile_images app Pillow # Image manipulation library; used for course assets, profile images, invoice PDFs, etc. +py2neo # Driver for converting Python modulestore structures to Neo4j's schema (for Coursegraph). pycountry pycryptodomex pygments # Used to support colors in paver command output diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 8609506bd8..8239e057ae 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -18,8 +18,6 @@ # via -r requirements/edx/github.in -e . # via -r requirements/edx/local.in --e git+https://github.com/technige/py2neo.git@py2neo-3.1.2#egg=py2neo==3.1.2 - # via -r requirements/edx/github.in -e git+https://github.com/edx/RateXBlock.git@2.0.1#egg=rate-xblock # via -r requirements/edx/github.in -e common/lib/safe_lxml @@ -105,6 +103,7 @@ certifi==2021.5.30 # via # -r requirements/edx/paver.txt # elasticsearch + # py2neo # requests cffi==1.14.6 # via cryptography @@ -123,7 +122,6 @@ click==7.1.2 # -c requirements/edx/../constraints.txt # code-annotations # nltk - # pact-python # user-util code-annotations==1.2.0 # via @@ -139,11 +137,12 @@ coreschema==0.0.4 # drf-yasg crowdsourcehinter-xblock==0.6 # via -r requirements/edx/base.in -cryptography==3.4.7 +cryptography==3.4.8 # via # -r requirements/edx/base.in # django-fernet-fields # edx-enterprise + # py2neo # pyjwt # social-auth-core cssutils==2.3.0 @@ -306,7 +305,7 @@ django-mptt==0.13.1 # django-wiki django-multi-email-field==0.6.2 # via edx-enterprise -django-mysql==3.12.0 +django-mysql==4.0.0 # via -r requirements/edx/base.in django-oauth-toolkit==1.3.2 # via @@ -381,6 +380,8 @@ djangorestframework==3.12.4 # super-csv djangorestframework-xml==2.0.0 # via edx-enterprise +docker==5.0.0 + # via py2neo docopt==0.6.2 # via xmodule docutils==0.17.1 @@ -513,6 +514,8 @@ elasticsearch==7.13.4 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # edx-search +english==2020.7.0 + # via py2neo enmerkar==0.7.1 # via enmerkar-underscore enmerkar-underscore==2.1.0 @@ -657,7 +660,9 @@ git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752 mongoengine==0.23.1 # via -r requirements/edx/base.in monotonic==1.6 - # via analytics-python + # via + # analytics-python + # py2neo mpmath==1.2.1 # via sympy multidict==5.1.0 @@ -666,6 +671,8 @@ multidict==5.1.0 # yarl mysqlclient==2.0.3 # via -r requirements/edx/base.in +neotime==1.7.4 + # via py2neo newrelic==6.8.0.163 # via # -r requirements/edx/base.in @@ -697,8 +704,9 @@ packaging==21.0 # via # bleach # drf-yasg -pact-python==1.3.9 - # via edxval + # py2neo +pansi==2020.7.3 + # via py2neo path==16.2.0 # via # -r requirements/edx/paver.txt @@ -725,11 +733,16 @@ pillow==8.3.1 # edx-organizations polib==1.1.1 # via edx-i18n-tools +prompt-toolkit==3.0.20 + # via py2neo psutil==5.8.0 # via # -r requirements/edx/paver.txt # edx-django-utils - # pact-python +py2neo==2021.1.5 + # via + # -c requirements/edx/../constraints.txt + # -r requirements/edx/base.in pycountry==20.7.3 # via -r requirements/edx/base.in pycparser==2.20 @@ -741,7 +754,9 @@ pycryptodomex==3.10.1 # lti-consumer-xblock # pyjwkest pygments==2.10.0 - # via -r requirements/edx/base.in + # via + # -r requirements/edx/base.in + # py2neo pyjwkest==1.4.2 # via # -r requirements/edx/base.in @@ -828,8 +843,10 @@ pytz==2021.1 # event-tracking # fs # icalendar + # neotime # olxcleaner # ora2 + # py2neo # tincan # xblock pyuca==1.2 @@ -855,6 +872,7 @@ requests==2.26.0 # analytics-python # coreapi # django-oauth-toolkit + # docker # edx-analytics-data-api-client # edx-bulk-grades # edx-drf-extensions @@ -863,7 +881,6 @@ requests==2.26.0 # edx-rest-api-client # geoip2 # mailsnake - # pact-python # pyjwkest # python-swiftclient # requests-oauthlib @@ -924,14 +941,17 @@ six==1.16.0 # edx-i18n-tools # edx-milestones # edx-rbac + # english # event-tracking # fs # fs-s3fs # html5lib # isodate # libsass - # pact-python + # neotime + # pansi # paver + # py2neo # pyjwkest # python-dateutil # python-memcached @@ -1011,6 +1031,7 @@ urllib3==1.26.6 # -r requirements/edx/paver.txt # elasticsearch # geoip2 + # py2neo # requests user-util==1.0.0 # via -r requirements/edx/base.in @@ -1022,6 +1043,8 @@ voluptuous==0.12.1 # via ora2 watchdog==2.1.5 # via -r requirements/edx/paver.txt +wcwidth==0.2.5 + # via prompt-toolkit web-fragments==1.1.0 # via # -r requirements/edx/base.in @@ -1038,6 +1061,8 @@ webob==1.8.7 # via # xblock # xmodule +websocket-client==1.2.1 + # via docker wrapt==1.11.2 # via # -c requirements/edx/../constraints.txt diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index b15138cf92..d2bf74594d 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -18,8 +18,6 @@ # via -r requirements/edx/testing.txt -e . # via -r requirements/edx/testing.txt --e git+https://github.com/technige/py2neo.git@py2neo-3.1.2#egg=py2neo==3.1.2 - # via -r requirements/edx/testing.txt -e git+https://github.com/edx/RateXBlock.git@2.0.1#egg=rate-xblock # via -r requirements/edx/testing.txt -e common/lib/safe_lxml @@ -57,6 +55,10 @@ appdirs==1.4.4 # via # -r requirements/edx/testing.txt # fs +asgiref==3.4.1 + # via + # -r requirements/edx/testing.txt + # uvicorn astroid==2.6.6 # via # -r requirements/edx/testing.txt @@ -136,6 +138,7 @@ certifi==2021.5.30 # via # -r requirements/edx/testing.txt # elasticsearch + # py2neo # requests cffi==1.14.6 # via @@ -165,6 +168,7 @@ click==7.1.2 # pact-python # pip-tools # user-util + # uvicorn click-log==0.3.2 # via # -r requirements/edx/testing.txt @@ -192,11 +196,12 @@ coverage==5.5 # pytest-cov crowdsourcehinter-xblock==0.6 # via -r requirements/edx/testing.txt -cryptography==3.4.7 +cryptography==3.4.8 # via # -r requirements/edx/testing.txt # django-fernet-fields # edx-enterprise + # py2neo # pyjwt # social-auth-core cssselect==1.1.0 @@ -386,7 +391,7 @@ django-multi-email-field==0.6.2 # via # -r requirements/edx/testing.txt # edx-enterprise -django-mysql==3.12.0 +django-mysql==4.0.0 # via -r requirements/edx/testing.txt django-oauth-toolkit==1.3.2 # via @@ -465,6 +470,10 @@ djangorestframework-xml==2.0.0 # via # -r requirements/edx/testing.txt # edx-enterprise +docker==5.0.0 + # via + # -r requirements/edx/testing.txt + # py2neo docopt==0.6.2 # via # -r requirements/edx/testing.txt @@ -617,6 +626,10 @@ elasticsearch==7.13.4 # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/edx/testing.txt # edx-search +english==2020.7.0 + # via + # -r requirements/edx/testing.txt + # py2neo enmerkar==0.7.1 # via # -r requirements/edx/testing.txt @@ -635,10 +648,14 @@ execnet==1.9.0 # pytest-xdist factory-boy==3.2.0 # via -r requirements/edx/testing.txt -faker==8.12.0 +faker==8.12.1 # via # -r requirements/edx/testing.txt # factory-boy +fastapi==0.68.1 + # via + # -r requirements/edx/testing.txt + # pact-python filelock==3.0.12 # via # -r requirements/edx/testing.txt @@ -671,7 +688,7 @@ gitdb==4.0.7 # via # -r requirements/edx/testing.txt # gitpython -gitpython==3.1.18 +gitpython==3.1.20 # via # -r requirements/edx/testing.txt # transifex-client @@ -679,6 +696,10 @@ glob2==0.7 # via -r requirements/edx/testing.txt gunicorn==20.1.0 # via -r requirements/edx/testing.txt +h11==0.12.0 + # via + # -r requirements/edx/testing.txt + # uvicorn help-tokens==2.1.0 # via -r requirements/edx/testing.txt html5lib==1.1 @@ -858,6 +879,7 @@ monotonic==1.6 # via # -r requirements/edx/testing.txt # analytics-python + # py2neo mpmath==1.2.1 # via # -r requirements/edx/testing.txt @@ -873,6 +895,10 @@ mypy-extensions==0.4.3 # via mypy mysqlclient==2.0.3 # via -r requirements/edx/testing.txt +neotime==1.7.4 + # via + # -r requirements/edx/testing.txt + # py2neo newrelic==6.8.0.163 # via # -r requirements/edx/testing.txt @@ -906,13 +932,16 @@ packaging==21.0 # -r requirements/edx/testing.txt # bleach # drf-yasg + # py2neo # pytest # sphinx # tox -pact-python==1.3.9 +pact-python==1.4.2 + # via -r requirements/edx/testing.txt +pansi==2020.7.3 # via # -r requirements/edx/testing.txt - # edxval + # py2neo path==16.2.0 # via # -r requirements/edx/testing.txt @@ -958,6 +987,10 @@ polib==1.1.1 # via # -r requirements/edx/testing.txt # edx-i18n-tools +prompt-toolkit==3.0.20 + # via + # -r requirements/edx/testing.txt + # py2neo psutil==5.8.0 # via # -r requirements/edx/testing.txt @@ -970,6 +1003,10 @@ py==1.10.0 # pytest # pytest-forked # tox +py2neo==2021.1.5 + # via + # -c requirements/edx/../constraints.txt + # -r requirements/edx/testing.txt pycodestyle==2.7.0 # via -r requirements/edx/testing.txt pycountry==20.7.3 @@ -984,10 +1021,15 @@ pycryptodomex==3.10.1 # edx-proctoring # lti-consumer-xblock # pyjwkest +pydantic==1.8.2 + # via + # -r requirements/edx/testing.txt + # fastapi pygments==2.10.0 # via # -r requirements/edx/testing.txt # diff-cover + # py2neo # sphinx pyjwkest==1.4.2 # via @@ -1143,8 +1185,10 @@ pytz==2021.1 # event-tracking # fs # icalendar + # neotime # olxcleaner # ora2 + # py2neo # tincan # xblock pyuca==1.2 @@ -1175,6 +1219,7 @@ requests==2.26.0 # analytics-python # coreapi # django-oauth-toolkit + # docker # edx-analytics-data-api-client # edx-bulk-grades # edx-drf-extensions @@ -1265,6 +1310,7 @@ six==1.16.0 # edx-milestones # edx-rbac # edx-sphinx-theme + # english # event-tracking # freezegun # fs @@ -1274,8 +1320,11 @@ six==1.16.0 # isodate # jsonschema # libsass + # neotime # pact-python + # pansi # paver + # py2neo # pyjwkest # python-dateutil # python-memcached @@ -1346,6 +1395,10 @@ sqlparse==0.4.1 # django-debug-toolbar staff-graded-xblock==1.5.1 # via -r requirements/edx/testing.txt +starlette==0.14.2 + # via + # -r requirements/edx/testing.txt + # fastapi stevedore==3.4.0 # via # -r requirements/edx/testing.txt @@ -1409,7 +1462,9 @@ typing-extensions==3.10.0.0 # via # -r requirements/edx/testing.txt # aiohttp + # gitpython # mypy + # pydantic ua-parser==0.10.0 # via # -r requirements/edx/testing.txt @@ -1430,11 +1485,17 @@ urllib3==1.26.6 # -r requirements/edx/testing.txt # elasticsearch # geoip2 + # pact-python + # py2neo # requests # selenium # transifex-client user-util==1.0.0 # via -r requirements/edx/testing.txt +uvicorn==0.15.0 + # via + # -r requirements/edx/testing.txt + # pact-python vine==1.3.0 # via # -r requirements/edx/testing.txt @@ -1452,6 +1513,10 @@ vulture==2.3 # via -r requirements/edx/development.in watchdog==2.1.5 # via -r requirements/edx/testing.txt +wcwidth==0.2.5 + # via + # -r requirements/edx/testing.txt + # prompt-toolkit web-fragments==1.1.0 # via # -r requirements/edx/testing.txt @@ -1470,6 +1535,10 @@ webob==1.8.7 # -r requirements/edx/testing.txt # xblock # xmodule +websocket-client==1.2.1 + # via + # -r requirements/edx/testing.txt + # docker wheel==0.37.0 # via # -r requirements/edx/pip-tools.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index bc5fe2c2f9..74d259eafb 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -24,7 +24,7 @@ edx-sphinx-theme==3.0.0 # via -r requirements/edx/doc.in gitdb==4.0.7 # via gitpython -gitpython==3.1.18 +gitpython==3.1.20 # via -r requirements/edx/doc.in idna==3.2 # via requests @@ -80,6 +80,8 @@ stevedore==3.4.0 # via code-annotations text-unidecode==1.3 # via python-slugify +typing-extensions==3.10.0.0 + # via gitpython urllib3==1.26.6 # via requests diff --git a/requirements/edx/github.in b/requirements/edx/github.in index b1d51c90e3..3ce2345581 100644 --- a/requirements/edx/github.in +++ b/requirements/edx/github.in @@ -58,7 +58,6 @@ -e git+https://github.com/openedx/olxcleaner.git@2f0d6c7f126cbd69c9724b7b57a0b2565330a297#egg=olxcleaner git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2 -e git+https://github.com/jazkarta/edx-jsme.git@690dbf75441fa91c7c4899df0b83d77f7deb5458#egg=edx-jsme --e git+https://github.com/technige/py2neo.git@py2neo-3.1.2#egg=py2neo==3.1.2 # This is a temporary fork until https://github.com/brutasse/django-ratelimit-backend/pull/50 is merged # back into the upstream code. diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 35529b784b..ceb223171e 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -18,8 +18,6 @@ # via -r requirements/edx/base.txt -e . # via -r requirements/edx/base.txt --e git+https://github.com/technige/py2neo.git@py2neo-3.1.2#egg=py2neo==3.1.2 - # via -r requirements/edx/base.txt -e git+https://github.com/edx/RateXBlock.git@2.0.1#egg=rate-xblock # via -r requirements/edx/base.txt -e common/lib/safe_lxml @@ -55,6 +53,8 @@ appdirs==1.4.4 # via # -r requirements/edx/base.txt # fs +asgiref==3.4.1 + # via uvicorn astroid==2.6.6 # via # pylint @@ -130,6 +130,7 @@ certifi==2021.5.30 # via # -r requirements/edx/base.txt # elasticsearch + # py2neo # requests cffi==1.14.6 # via @@ -156,6 +157,7 @@ click==7.1.2 # nltk # pact-python # user-util + # uvicorn click-log==0.3.2 # via edx-lint code-annotations==1.2.0 @@ -182,11 +184,12 @@ coverage==5.5 # pytest-cov crowdsourcehinter-xblock==0.6 # via -r requirements/edx/base.txt -cryptography==3.4.7 +cryptography==3.4.8 # via # -r requirements/edx/base.txt # django-fernet-fields # edx-enterprise + # py2neo # pyjwt # social-auth-core cssselect==1.1.0 @@ -371,7 +374,7 @@ django-multi-email-field==0.6.2 # via # -r requirements/edx/base.txt # edx-enterprise -django-mysql==3.12.0 +django-mysql==4.0.0 # via -r requirements/edx/base.txt django-oauth-toolkit==1.3.2 # via @@ -450,6 +453,10 @@ djangorestframework-xml==2.0.0 # via # -r requirements/edx/base.txt # edx-enterprise +docker==5.0.0 + # via + # -r requirements/edx/base.txt + # py2neo docopt==0.6.2 # via # -r requirements/edx/base.txt @@ -599,6 +606,10 @@ elasticsearch==7.13.4 # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/edx/base.txt # edx-search +english==2020.7.0 + # via + # -r requirements/edx/base.txt + # py2neo enmerkar==0.7.1 # via # -r requirements/edx/base.txt @@ -615,8 +626,10 @@ execnet==1.9.0 # via pytest-xdist factory-boy==3.2.0 # via -r requirements/edx/testing.in -faker==8.12.0 +faker==8.12.1 # via factory-boy +fastapi==0.68.1 + # via pact-python filelock==3.0.12 # via # tox @@ -646,12 +659,14 @@ geoip2==4.2.0 # via -r requirements/edx/base.txt gitdb==4.0.7 # via gitpython -gitpython==3.1.18 +gitpython==3.1.20 # via transifex-client glob2==0.7 # via -r requirements/edx/base.txt gunicorn==20.1.0 # via -r requirements/edx/base.txt +h11==0.12.0 + # via uvicorn help-tokens==2.1.0 # via -r requirements/edx/base.txt html5lib==1.1 @@ -816,6 +831,7 @@ monotonic==1.6 # via # -r requirements/edx/base.txt # analytics-python + # py2neo mpmath==1.2.1 # via # -r requirements/edx/base.txt @@ -827,6 +843,10 @@ multidict==5.1.0 # yarl mysqlclient==2.0.3 # via -r requirements/edx/base.txt +neotime==1.7.4 + # via + # -r requirements/edx/base.txt + # py2neo newrelic==6.8.0.163 # via # -r requirements/edx/base.txt @@ -860,13 +880,15 @@ packaging==21.0 # -r requirements/edx/base.txt # bleach # drf-yasg + # py2neo # pytest # tox -pact-python==1.3.9 +pact-python==1.4.2 + # via -r requirements/edx/testing.in +pansi==2020.7.3 # via # -r requirements/edx/base.txt - # -r requirements/edx/testing.in - # edxval + # py2neo path==16.2.0 # via # -r requirements/edx/base.txt @@ -905,6 +927,10 @@ polib==1.1.1 # -r requirements/edx/base.txt # -r requirements/edx/testing.in # edx-i18n-tools +prompt-toolkit==3.0.20 + # via + # -r requirements/edx/base.txt + # py2neo psutil==5.8.0 # via # -r requirements/edx/base.txt @@ -916,6 +942,10 @@ py==1.10.0 # pytest # pytest-forked # tox +py2neo==2021.1.5 + # via + # -c requirements/edx/../constraints.txt + # -r requirements/edx/base.txt pycodestyle==2.7.0 # via -r requirements/edx/testing.in pycountry==20.7.3 @@ -930,11 +960,14 @@ pycryptodomex==3.10.1 # edx-proctoring # lti-consumer-xblock # pyjwkest +pydantic==1.8.2 + # via fastapi pygments==2.10.0 # via # -r requirements/edx/base.txt # -r requirements/edx/coverage.txt # diff-cover + # py2neo pyjwkest==1.4.2 # via # -r requirements/edx/base.txt @@ -1079,8 +1112,10 @@ pytz==2021.1 # event-tracking # fs # icalendar + # neotime # olxcleaner # ora2 + # py2neo # tincan # xblock pyuca==1.2 @@ -1108,6 +1143,7 @@ requests==2.26.0 # analytics-python # coreapi # django-oauth-toolkit + # docker # edx-analytics-data-api-client # edx-bulk-grades # edx-drf-extensions @@ -1196,6 +1232,7 @@ six==1.16.0 # edx-lint # edx-milestones # edx-rbac + # english # event-tracking # freezegun # fs @@ -1204,8 +1241,11 @@ six==1.16.0 # httpretty # isodate # libsass + # neotime # pact-python + # pansi # paver + # py2neo # pyjwkest # python-dateutil # python-memcached @@ -1250,6 +1290,8 @@ sqlparse==0.4.1 # django staff-graded-xblock==1.5.1 # via -r requirements/edx/base.txt +starlette==0.14.2 + # via fastapi stevedore==3.4.0 # via # -r requirements/edx/base.txt @@ -1307,6 +1349,8 @@ typing-extensions==3.10.0.0 # via # -r requirements/edx/base.txt # aiohttp + # gitpython + # pydantic ua-parser==0.10.0 # via # -r requirements/edx/base.txt @@ -1327,11 +1371,15 @@ urllib3==1.26.6 # -r requirements/edx/base.txt # elasticsearch # geoip2 + # pact-python + # py2neo # requests # selenium # transifex-client user-util==1.0.0 # via -r requirements/edx/base.txt +uvicorn==0.15.0 + # via pact-python vine==1.3.0 # via # -r requirements/edx/base.txt @@ -1345,6 +1393,10 @@ voluptuous==0.12.1 # ora2 watchdog==2.1.5 # via -r requirements/edx/base.txt +wcwidth==0.2.5 + # via + # -r requirements/edx/base.txt + # prompt-toolkit web-fragments==1.1.0 # via # -r requirements/edx/base.txt @@ -1363,6 +1415,10 @@ webob==1.8.7 # -r requirements/edx/base.txt # xblock # xmodule +websocket-client==1.2.1 + # via + # -r requirements/edx/base.txt + # docker wrapt==1.11.2 # via # -c requirements/edx/../constraints.txt