From 64722bfc8a02fde5fbc87b99fa92c5ed4210d88a Mon Sep 17 00:00:00 2001 From: Adam Palay Date: Mon, 14 Nov 2016 17:54:01 -0500 Subject: [PATCH] Moves dump_to_neo4j cache backend to neo4j instead of memcached (SUST-76) Also * Instead of caching when a course is last published, we get this information from the CourseStructure table * This commit introduces a mock py2neo Graph to be used for testing --- openedx/core/djangoapps/coursegraph/apps.py | 6 - .../management/commands/dump_to_neo4j.py | 134 +++++++---- .../commands/tests/test_dump_to_neo4j.py | 215 +++++++++++------- .../management/commands/tests/utils.py | 123 ++++++++++ .../core/djangoapps/coursegraph/signals.py | 15 -- .../djangoapps/coursegraph/tests/__init__.py | 0 .../coursegraph/tests/test_signals.py | 27 --- openedx/core/djangoapps/coursegraph/utils.py | 52 ----- 8 files changed, 347 insertions(+), 225 deletions(-) create mode 100644 openedx/core/djangoapps/coursegraph/management/commands/tests/utils.py delete mode 100644 openedx/core/djangoapps/coursegraph/signals.py delete mode 100644 openedx/core/djangoapps/coursegraph/tests/__init__.py delete mode 100644 openedx/core/djangoapps/coursegraph/tests/test_signals.py delete mode 100644 openedx/core/djangoapps/coursegraph/utils.py diff --git a/openedx/core/djangoapps/coursegraph/apps.py b/openedx/core/djangoapps/coursegraph/apps.py index 1c49ead924..11524aa240 100644 --- a/openedx/core/djangoapps/coursegraph/apps.py +++ b/openedx/core/djangoapps/coursegraph/apps.py @@ -12,9 +12,3 @@ class CoursegraphConfig(AppConfig): AppConfig for courseware app """ name = 'openedx.core.djangoapps.coursegraph' - - def ready(self): - """ - Import signals on startup - """ - from openedx.core.djangoapps.coursegraph import signals # pylint: disable=unused-variable diff --git a/openedx/core/djangoapps/coursegraph/management/commands/dump_to_neo4j.py b/openedx/core/djangoapps/coursegraph/management/commands/dump_to_neo4j.py index c1ac49e1e0..aab19ca9a9 100644 --- a/openedx/core/djangoapps/coursegraph/management/commands/dump_to_neo4j.py +++ b/openedx/core/djangoapps/coursegraph/management/commands/dump_to_neo4j.py @@ -7,18 +7,14 @@ from __future__ import unicode_literals, print_function import logging from django.core.management.base import BaseCommand -from django.utils import six +from django.utils import six, timezone from opaque_keys.edx.keys import CourseKey -from py2neo import Graph, Node, Relationship, authenticate +from py2neo import Graph, Node, Relationship, authenticate, NodeSelector from py2neo.compat import integer, string, unicode as neo4j_unicode from request_cache.middleware import RequestCache from xmodule.modulestore.django import modulestore -from openedx.core.djangoapps.coursegraph.utils import ( - CommandLastRunCache, - CourseLastPublishedCache, -) - +from openedx.core.djangoapps.content.course_structures.models import CourseStructure log = logging.getLogger(__name__) @@ -29,9 +25,6 @@ bolt_log.setLevel(logging.ERROR) PRIMITIVE_NEO4J_TYPES = (integer, string, neo4j_unicode, float, bool) -COMMAND_LAST_RUN_CACHE = CommandLastRunCache() -COURSE_LAST_PUBLISHED_CACHE = CourseLastPublishedCache() - class ModuleStoreSerializer(object): """ @@ -45,8 +38,10 @@ class ModuleStoreSerializer(object): If that parameter isn't furnished, loads all course_keys from the modulestore. Filters out course_keys in the `skip` parameter, if provided. - :param courses: string serialization of course keys - :param skip: string serialization of course keys + Args: + courses: A list of string serializations of course keys. + For example, ["course-v1:org+course+run"]. + skip: Also a list of string serializations of course keys. """ if courses: course_keys = [CourseKey.from_string(course.strip()) for course in courses] @@ -67,7 +62,7 @@ class ModuleStoreSerializer(object): Returns: fields: a dictionary of an XBlock's field names and values - label: the name of the XBlock's type (i.e. 'course' + block_type: the name of the XBlock's type (i.e. 'course' or 'problem') """ # convert all fields to a dict and filter out parent and children field @@ -88,25 +83,27 @@ class ModuleStoreSerializer(object): fields['course_key'] = six.text_type(course_key) fields['location'] = six.text_type(item.location) - label = item.scope_ids.block_type + block_type = item.scope_ids.block_type - # prune some fields - if label == 'course': + if block_type == 'course': + # prune the checklists field if 'checklists' in fields: del fields['checklists'] - return fields, label + # record the time this command was run + fields['time_last_dumped_to_neo4j'] = six.text_type(timezone.now()) + + return fields, block_type def serialize_course(self, course_id): """ + Serializes a course into py2neo Nodes and Relationships Args: course_id: CourseKey of the course we want to serialize Returns: nodes: a list of py2neo Node objects relationships: a list of py2neo Relationships objects - - Serializes a course into Nodes and Relationships """ # create a location to node mapping we'll need later for # writing relationships @@ -116,12 +113,12 @@ class ModuleStoreSerializer(object): # create nodes nodes = [] for item in items: - fields, label = self.serialize_item(item) + fields, block_type = self.serialize_item(item) for field_name, value in six.iteritems(fields): fields[field_name] = self.coerce_types(value) - node = Node(label, 'item', **fields) + node = Node(block_type, 'item', **fields) nodes.append(node) location_to_node[item.location] = node @@ -144,7 +141,7 @@ class ModuleStoreSerializer(object): value: the value of an xblock's field Returns: either the value, a text version of the value, or, if the - value is a list, a list where each element is converted to text. + value is a list, a list where each element is converted to text. """ coerced_value = value if isinstance(value, list): @@ -168,44 +165,92 @@ class ModuleStoreSerializer(object): transaction.create(entity) @staticmethod - def should_dump_course(course_key): + def get_command_last_run(course_key, graph): + """ + This information is stored on the course node of a course in neo4j + Args: + course_key: a CourseKey + graph: a py2neo Graph + + Returns: The datetime that the command was last run, converted into + text, or None, if there's no record of this command last being run. + + """ + selector = NodeSelector(graph) + course_node = selector.select( + "course", + course_key=six.text_type(course_key) + ).first() + + last_this_command_was_run = None + if course_node: + last_this_command_was_run = course_node['time_last_dumped_to_neo4j'] + + return last_this_command_was_run + + @staticmethod + def get_course_last_published(course_key): + """ + We use the CourseStructure table to get when this course was last + published. + Args: + course_key: a CourseKey + + Returns: The datetime the course was last published at, converted into + text, or None, if there's no record of the last time this course + was published. + """ + try: + structure = CourseStructure.objects.get(course_id=course_key) + course_last_published_date = six.text_type(structure.modified) + except CourseStructure.DoesNotExist: + course_last_published_date = None + + return course_last_published_date + + def should_dump_course(self, course_key, graph): """ Only dump the course if it's been changed since the last time it's been dumped. - :param course_key: a CourseKey object. - :return: bool. Whether or not this course should be dumped to neo4j. + Args: + course_key: a CourseKey object. + graph: a py2neo Graph object. + + Returns: bool of whether this course should be dumped to neo4j. """ - last_this_command_was_run = COMMAND_LAST_RUN_CACHE.get(course_key) - last_course_had_published_event = COURSE_LAST_PUBLISHED_CACHE.get( - course_key - ) + last_this_command_was_run = self.get_command_last_run(course_key, graph) - # if we have no record of this course being serialized, serialize it + course_last_published_date = self.get_course_last_published(course_key) + + # if we don't have a record of the last time this command was run, + # we should serialize the course and dump it if last_this_command_was_run is None: return True # if we've serialized the course recently and we have no published - # events, we can skip re-serializing it - if last_this_command_was_run and last_course_had_published_event is None: + # events, we will not dump it, and so we can skip serializing it + # again here + if last_this_command_was_run and course_last_published_date is None: return False - # otherwise, serialize if the command was run before the course's last - # published event - return last_this_command_was_run < last_course_had_published_event + # otherwise, serialize and dump the course if the command was run + # before the course's last published event + return last_this_command_was_run < course_last_published_date def dump_courses_to_neo4j(self, graph, override_cache=False): """ - Parameters - ---------- - graph: py2neo graph object - override_cache: serialize the courses even if they'be been recently - serialized + Method that iterates through a list of courses in a modulestore, + serializes them, then writes them to neo4j + Args: + graph: py2neo graph object + override_cache: serialize the courses even if they'be been recently + serialized - Returns two lists: one of the courses that were successfully written - to neo4j, and one of courses that were not. - ------- + Returns: two lists--one of the courses that were successfully written + to neo4j and one of courses that were not. """ + total_number_of_courses = len(self.course_keys) successful_courses = [] @@ -222,7 +267,7 @@ class ModuleStoreSerializer(object): total_number_of_courses, ) - if not (override_cache or self.should_dump_course(course_key)): + if not (override_cache or self.should_dump_course(course_key, graph)): log.info("skipping dumping %s, since it hasn't changed", course_key) continue @@ -258,7 +303,6 @@ class ModuleStoreSerializer(object): unsuccessful_courses.append(course_string) else: - COMMAND_LAST_RUN_CACHE.set(course_key) successful_courses.append(course_string) return successful_courses, unsuccessful_courses diff --git a/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py b/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py index d92bef636e..1d41ec1334 100644 --- a/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py +++ b/openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py @@ -16,7 +16,13 @@ from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory from openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j import ( ModuleStoreSerializer, ) -from openedx.core.djangoapps.coursegraph.signals import _listen_for_course_publish +from openedx.core.djangoapps.coursegraph.management.commands.tests.utils import ( + MockGraph, + MockNodeSelector, +) +from openedx.core.djangoapps.content.course_structures.signals import ( + listen_for_course_publish +) class TestDumpToNeo4jCommandBase(SharedModuleStoreTestCase): @@ -39,6 +45,43 @@ class TestDumpToNeo4jCommandBase(SharedModuleStoreTestCase): cls.course_strings = [six.text_type(cls.course.id), six.text_type(cls.course2.id)] + @staticmethod + def setup_mock_graph(mock_selector_class, mock_graph_class, transaction_errors=False): + """ + Replaces the py2neo Graph object with a MockGraph; similarly replaces + NodeSelector with MockNodeSelector. + + Args: + mock_selector_class: a mocked NodeSelector class + mock_graph_class: a mocked Graph class + transaction_errors: a bool for whether we should get errors + when transactions try to commit + + Returns: an instance of MockGraph + """ + + mock_graph = MockGraph(transaction_errors=transaction_errors) + mock_graph_class.return_value = mock_graph + + mock_node_selector = MockNodeSelector(mock_graph) + mock_selector_class.return_value = mock_node_selector + return mock_graph + + def assertCourseDump(self, mock_graph, number_of_courses, number_commits, number_rollbacks): + """ + Asserts that we have the expected number of courses, commits, and + rollbacks after we dump the modulestore to neo4j + Args: + mock_graph: a MockGraph backend + number_of_courses: number of courses we expect to find + number_commits: number of commits we expect against the graph + number_rollbacks: number of commit rollbacks we expect + """ + courses = set([node['course_key'] for node in mock_graph.nodes]) + self.assertEqual(len(courses), number_of_courses) + self.assertEqual(mock_graph.number_commits, number_commits) + self.assertEqual(mock_graph.number_rollbacks, number_rollbacks) + @ddt.ddt class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): @@ -46,15 +89,14 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): Tests for the dump to neo4j management command """ + @mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.NodeSelector') @mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.Graph') @ddt.data(1, 2) - def test_dump_specific_courses(self, number_of_courses, mock_graph_class): + def test_dump_specific_courses(self, number_of_courses, mock_graph_class, mock_selector_class): """ Test that you can specify which courses you want to dump. """ - mock_graph = mock_graph_class.return_value - mock_transaction = mock.Mock() - mock_graph.begin.return_value = mock_transaction + mock_graph = self.setup_mock_graph(mock_selector_class, mock_graph_class) call_command( 'dump_to_neo4j', @@ -65,18 +107,22 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): password='mock_password', ) - self.assertEqual(mock_graph.begin.call_count, number_of_courses) - self.assertEqual(mock_transaction.commit.call_count, number_of_courses) - self.assertEqual(mock_transaction.commit.rollback.call_count, 0) + self.assertCourseDump( + mock_graph, + number_of_courses=number_of_courses, + number_commits=number_of_courses, + number_rollbacks=0 + ) + @mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.NodeSelector') @mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.Graph') - def test_dump_skip_course(self, mock_graph_class): + def test_dump_skip_course(self, mock_graph_class, mock_selector_class): """ Test that you can skip courses. """ - mock_graph = mock_graph_class.return_value - mock_transaction = mock.Mock() - mock_graph.begin.return_value = mock_transaction + mock_graph = self.setup_mock_graph( + mock_selector_class, mock_graph_class + ) call_command( 'dump_to_neo4j', @@ -87,18 +133,22 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): password='mock_password', ) - self.assertEqual(mock_graph.begin.call_count, 1) - self.assertEqual(mock_transaction.commit.call_count, 1) - self.assertEqual(mock_transaction.commit.rollback.call_count, 0) + self.assertCourseDump( + mock_graph, + number_of_courses=1, + number_commits=1, + number_rollbacks=0, + ) + @mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.NodeSelector') @mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.Graph') - def test_dump_skip_beats_specifying(self, mock_graph_class): + def test_dump_skip_beats_specifying(self, mock_graph_class, mock_selector_class): """ Test that if you skip and specify the same course, you'll skip it. """ - mock_graph = mock_graph_class.return_value - mock_transaction = mock.Mock() - mock_graph.begin.return_value = mock_transaction + mock_graph = self.setup_mock_graph( + mock_selector_class, mock_graph_class + ) call_command( 'dump_to_neo4j', @@ -110,31 +160,38 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase): password='mock_password', ) - self.assertEqual(mock_graph.begin.call_count, 0) - self.assertEqual(mock_transaction.commit.call_count, 0) - self.assertEqual(mock_transaction.commit.rollback.call_count, 0) + self.assertCourseDump( + mock_graph, + number_of_courses=0, + number_commits=0, + number_rollbacks=0, + ) + @mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.NodeSelector') @mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.Graph') - def test_dump_all_courses(self, mock_graph_class): + def test_dump_all_courses(self, mock_graph_class, mock_selector_class): """ Test if you don't specify which courses to dump, then you'll dump all of them. """ - mock_graph = mock_graph_class.return_value - mock_transaction = mock.Mock() - mock_graph.begin.return_value = mock_transaction + mock_graph = self.setup_mock_graph( + mock_selector_class, mock_graph_class + ) call_command( 'dump_to_neo4j', host='mock_host', http_port=7474, user='mock_user', - password='mock_password', + password='mock_password' ) - self.assertEqual(mock_graph.begin.call_count, 2) - self.assertEqual(mock_transaction.commit.call_count, 2) - self.assertEqual(mock_transaction.commit.rollback.call_count, 0) + self.assertCourseDump( + mock_graph, + number_of_courses=2, + number_commits=2, + number_rollbacks=0, + ) @ddt.ddt @@ -167,9 +224,7 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): """ Tests the serialize_course method. """ - nodes, relationships = self.mss.serialize_course( - self.course.id - ) + nodes, relationships = self.mss.serialize_course(self.course.id) self.assertEqual(len(nodes), 9) self.assertEqual(len(relationships), 7) @@ -194,63 +249,68 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): coerced_value = self.mss.coerce_types(original_value) self.assertEqual(coerced_value, coerced_expected) - def test_dump_to_neo4j(self): + @mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.NodeSelector') + def test_dump_to_neo4j(self, mock_selector_class): """ Tests the dump_to_neo4j method works against a mock py2neo Graph """ - mock_graph = mock.Mock() - mock_transaction = mock.Mock() - mock_graph.begin.return_value = mock_transaction + mock_graph = MockGraph() + mock_selector_class.return_value = MockNodeSelector(mock_graph) successful, unsuccessful = self.mss.dump_courses_to_neo4j(mock_graph) - self.assertEqual(mock_graph.begin.call_count, 2) - self.assertEqual(mock_transaction.commit.call_count, 2) - self.assertEqual(mock_transaction.rollback.call_count, 0) + self.assertCourseDump( + mock_graph, + number_of_courses=2, + number_commits=2, + number_rollbacks=0, + ) - # 7 nodes + 9 relationships from the first course + # 9 nodes + 7 relationships from the first course # 2 nodes and no relationships from the second - self.assertEqual(mock_transaction.create.call_count, 18) - self.assertEqual(mock_transaction.run.call_count, 2) + + self.assertEqual(len(mock_graph.nodes), 11) self.assertEqual(len(unsuccessful), 0) self.assertItemsEqual(successful, self.course_strings) - def test_dump_to_neo4j_rollback(self): + @mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.NodeSelector') + def test_dump_to_neo4j_rollback(self, mock_selector_class): """ Tests that the the dump_to_neo4j method handles the case where there's an exception trying to write to the neo4j database. """ - mock_graph = mock.Mock() - mock_transaction = mock.Mock() - mock_graph.begin.return_value = mock_transaction - mock_transaction.run.side_effect = ValueError('Something went wrong!') + mock_graph = MockGraph(transaction_errors=True) + mock_selector_class.return_value = MockNodeSelector(mock_graph) successful, unsuccessful = self.mss.dump_courses_to_neo4j(mock_graph) - self.assertEqual(mock_graph.begin.call_count, 2) - self.assertEqual(mock_transaction.commit.call_count, 0) - self.assertEqual(mock_transaction.rollback.call_count, 2) + self.assertCourseDump( + mock_graph, + number_of_courses=0, + number_commits=0, + number_rollbacks=2, + ) self.assertEqual(len(successful), 0) self.assertItemsEqual(unsuccessful, self.course_strings) - @ddt.data( - (True, 2), - (False, 0), - ) + @mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.NodeSelector') + @ddt.data((True, 2), (False, 0)) @ddt.unpack - def test_dump_to_neo4j_cache(self, override_cache, expected_number_courses): + def test_dump_to_neo4j_cache(self, override_cache, expected_number_courses, mock_selector_class): """ Tests the caching mechanism and override to make sure we only publish recently updated courses. """ - mock_graph = mock.Mock() + mock_graph = MockGraph() + mock_selector_class.return_value = MockNodeSelector(mock_graph) # run once to warm the cache - successful, unsuccessful = self.mss.dump_courses_to_neo4j(mock_graph) - self.assertEqual(len(successful + unsuccessful), len(self.course_strings)) + self.mss.dump_courses_to_neo4j( + mock_graph, override_cache=override_cache + ) # when run the second time, only dump courses if the cache override # is enabled @@ -259,19 +319,21 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): ) self.assertEqual(len(successful + unsuccessful), expected_number_courses) - def test_dump_to_neo4j_published(self): + @mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.NodeSelector') + def test_dump_to_neo4j_published(self, mock_selector_class): """ Tests that we only dump those courses that have been published after the last time the command was been run. """ - mock_graph = mock.Mock() + mock_graph = MockGraph() + mock_selector_class.return_value = MockNodeSelector(mock_graph) # run once to warm the cache successful, unsuccessful = self.mss.dump_courses_to_neo4j(mock_graph) self.assertEqual(len(successful + unsuccessful), len(self.course_strings)) # simulate one of the courses being published - _listen_for_course_publish(None, self.course.id) + listen_for_course_publish(None, self.course.id) # make sure only the published course was dumped successful, unsuccessful = self.mss.dump_courses_to_neo4j(mock_graph) @@ -280,31 +342,24 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase): self.assertEqual(successful[0], unicode(self.course.id)) @ddt.data( - (datetime(2016, 3, 30), datetime(2016, 3, 31), True), - (datetime(2016, 3, 31), datetime(2016, 3, 30), False), - (datetime(2016, 3, 31), None, False), - (None, datetime(2016, 3, 30), True), + (six.text_type(datetime(2016, 3, 30)), six.text_type(datetime(2016, 3, 31)), True), + (six.text_type(datetime(2016, 3, 31)), six.text_type(datetime(2016, 3, 30)), False), + (six.text_type(datetime(2016, 3, 31)), None, False), + (None, six.text_type(datetime(2016, 3, 30)), True), (None, None, True), ) @ddt.unpack - @mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.COMMAND_LAST_RUN_CACHE') - @mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.COURSE_LAST_PUBLISHED_CACHE') - def test_should_dump_course( - self, - last_command_run, - last_course_published, - should_dump, - mock_course_last_published_cache, - mock_command_last_run_cache, - ): + def test_should_dump_course(self, last_command_run, last_course_published, should_dump): """ Tests whether a course should be dumped given the last time it was dumped and the last time it was published. """ - mock_command_last_run_cache.get.return_value = last_command_run - mock_course_last_published_cache.get.return_value = last_course_published + mss = ModuleStoreSerializer() + mss.get_command_last_run = lambda course_key, graph: last_command_run + mss.get_course_last_published = lambda course_key: last_course_published mock_course_key = mock.Mock + mock_graph = mock.Mock() self.assertEqual( - self.mss.should_dump_course(mock_course_key), - should_dump + mss.should_dump_course(mock_course_key, mock_graph), + should_dump, ) diff --git a/openedx/core/djangoapps/coursegraph/management/commands/tests/utils.py b/openedx/core/djangoapps/coursegraph/management/commands/tests/utils.py new file mode 100644 index 0000000000..4a963baa8b --- /dev/null +++ b/openedx/core/djangoapps/coursegraph/management/commands/tests/utils.py @@ -0,0 +1,123 @@ +""" +Utilities for testing the dump_to_neo4j management command +""" +from __future__ import unicode_literals + +from py2neo import Node + + +class MockGraph(object): + """ + A stubbed out version of py2neo's Graph object, used for testing. + Args: + transaction_errors: a bool for whether transactions should throw + an error. + """ + def __init__(self, transaction_errors=False, **kwargs): # pylint: disable=unused-argument + self.nodes = set() + self.number_commits = 0 + self.number_rollbacks = 0 + self.transaction_errors = transaction_errors + + def begin(self): + """ + A stub of the method that generates transactions + Returns: a MockTransaction object (instead of a py2neo Transaction) + """ + return MockTransaction(self) + + +class MockTransaction(object): + """ + A stubbed out version of py2neo's Transaction object, used for testing. + """ + def __init__(self, graph): + self.temp = set() + self.graph = graph + + def run(self, query): + """ + Deletes all nodes associated with a course. Normally `run` executes + an arbitrary query, but in our code, we only use it to delete nodes + associated with a course. + Args: + query: query string to be executed (in this case, to delete all + nodes associated with a course) + """ + start_string = "WHERE n.course_key='" + start = query.index(start_string) + len(start_string) + query = query[start:] + end = query.find("'") + course_key = query[:end] + + self.graph.nodes = set([ + node for node in self.graph.nodes if node['course_key'] != course_key + ]) + + def create(self, element): + """ + Adds elements to the transaction's temporary backend storage + Args: + element: a py2neo Node object + """ + if isinstance(element, Node): + self.temp.add(element) + + def commit(self): + """ + Takes elements in the transaction's temporary storage and adds them + to the mock graph's storage. Throws an error if the graph's + transaction_errors param is set to True. + """ + if self.graph.transaction_errors: + raise Exception("fake exception while trying to commit") + for element in self.temp: + self.graph.nodes.add(element) + self.temp.clear() + self.graph.number_commits += 1 + + def rollback(self): + """ + Clears the transactions temporary storage + """ + self.temp.clear() + self.graph.number_rollbacks += 1 + + +class MockNodeSelector(object): + """ + Mocks out py2neo's NodeSelector class. Used to select a node from a graph. + py2neo's NodeSelector expects a real graph object to run queries against, + so, rather than have to mock out MockGraph to accommodate those queries, + it seemed simpler to mock out NodeSelector as well. + """ + def __init__(self, graph): + self.graph = graph + + def select(self, label, course_key): + """ + Selects nodes that match a label and course_key + Args: + label: the string of the label we're selecting nodes by + course_key: the string of the course key we're selecting node by + + Returns: a MockResult of matching nodes + """ + nodes = [] + for node in self.graph.nodes: + if node.has_label(label) and node["course_key"] == course_key: + nodes.append(node) + return MockNodeSelection(nodes) + + +class MockNodeSelection(list): + """ + Mocks out py2neo's NodeSelection class: this is the type of what + MockNodeSelector's `select` method returns. + """ + def first(self): + """ + Returns: the first element of a list if the list has elements. + Otherwise, None. + """ + return self[0] if self else None diff --git a/openedx/core/djangoapps/coursegraph/signals.py b/openedx/core/djangoapps/coursegraph/signals.py deleted file mode 100644 index 19522b5eb8..0000000000 --- a/openedx/core/djangoapps/coursegraph/signals.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -Signal handlers for the CourseGraph application -""" -from django.dispatch.dispatcher import receiver -from xmodule.modulestore.django import SignalHandler - -from openedx.core.djangoapps.coursegraph.utils import CourseLastPublishedCache - - -@receiver(SignalHandler.course_published) -def _listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable=unused-argument - """ - Register when the course was published on a course publish event - """ - CourseLastPublishedCache().set(course_key) diff --git a/openedx/core/djangoapps/coursegraph/tests/__init__.py b/openedx/core/djangoapps/coursegraph/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/openedx/core/djangoapps/coursegraph/tests/test_signals.py b/openedx/core/djangoapps/coursegraph/tests/test_signals.py deleted file mode 100644 index 838405e468..0000000000 --- a/openedx/core/djangoapps/coursegraph/tests/test_signals.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -Tests for coursegraph's signal handler on course publish -""" -from __future__ import unicode_literals - -from opaque_keys.edx.keys import CourseKey - -from openedx.core.djangoapps.coursegraph.signals import _listen_for_course_publish -from openedx.core.djangoapps.coursegraph.utils import CourseLastPublishedCache -from openedx.core.djangolib.testing.utils import CacheIsolationTestCase - - -class TestCourseGraphSignalHandler(CacheIsolationTestCase): - """ - Tests for the course publish course handler - """ - ENABLED_CACHES = ['default'] - - def test_cache_set_on_course_publish(self): - """ - Tests that the last published cache is set on course publish - """ - course_key = CourseKey.from_string('course-v1:org+course+run') - last_published_cache = CourseLastPublishedCache() - self.assertIsNone(last_published_cache.get(course_key)) - _listen_for_course_publish(None, course_key) - self.assertIsNotNone(last_published_cache.get(course_key)) diff --git a/openedx/core/djangoapps/coursegraph/utils.py b/openedx/core/djangoapps/coursegraph/utils.py deleted file mode 100644 index d55743f924..0000000000 --- a/openedx/core/djangoapps/coursegraph/utils.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -Helpers for the CourseGraph app -""" -from django.core.cache import cache -from django.utils import timezone - - -class TimeRecordingCacheBase(object): - """ - A base class for caching the current time for some key. - """ - # cache_prefix should be defined in children classes - cache_prefix = None - _cache = cache - - def _key(self, course_key): - """ - Make a cache key from the prefix and a course_key - :param course_key: CourseKey object - :return: a cache key - """ - return self.cache_prefix + unicode(course_key) - - def get(self, course_key): - """ - Gets the time value associated with the CourseKey. - :param course_key: a CourseKey object. - :return: the time the key was last set. - """ - return self._cache.get(self._key(course_key)) - - def set(self, course_key): - """ - Sets the current time for a CourseKey key. - :param course_key: a CourseKey object. - """ - return self._cache.set(self._key(course_key), timezone.now()) - - -class CourseLastPublishedCache(TimeRecordingCacheBase): - """ - Used to record the last time that a course had a publish event run on it. - """ - cache_prefix = u'course_last_published' - - -class CommandLastRunCache(TimeRecordingCacheBase): - """ - Used to record the last time that the dump_to_neo4j command was run on a - course. - """ - cache_prefix = u'dump_to_neo4j_command_last_run'