Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Moves dump_to_neo4j cache backend to neo4j instead of memcached (SUST-76)

Browse Source 
Also
* Instead of caching when a course is last published, we get this information
  from the CourseStructure table
* This commit introduces a mock py2neo Graph to be used for testing
This commit is contained in:
Adam Palay
2016-11-14 17:54:01 -05:00
parent 9749bbab0c
commit 64722bfc8a
8 changed files with 347 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
openedx/core/djangoapps/coursegraph/apps.py
View File

@@ -12,9 +12,3 @@ class CoursegraphConfig(AppConfig):
AppConfig for courseware app
"""
name = 'openedx.core.djangoapps.coursegraph'
def ready(self):
"""
Import signals on startup
"""
from openedx.core.djangoapps.coursegraph import signals # pylint: disable=unused-variable

134
openedx/core/djangoapps/coursegraph/management/commands/dump_to_neo4j.py
View File

@@ -7,18 +7,14 @@ from __future__ import unicode_literals, print_function
import logging
from django.core.management.base import BaseCommand
from django.utils import six
from django.utils import six, timezone
from opaque_keys.edx.keys import CourseKey
from py2neo import Graph, Node, Relationship, authenticate
from py2neo import Graph, Node, Relationship, authenticate, NodeSelector
from py2neo.compat import integer, string, unicode as neo4j_unicode
from request_cache.middleware import RequestCache
from xmodule.modulestore.django import modulestore
from openedx.core.djangoapps.coursegraph.utils import (
CommandLastRunCache,
CourseLastPublishedCache,
)
from openedx.core.djangoapps.content.course_structures.models import CourseStructure
log = logging.getLogger(__name__)
@@ -29,9 +25,6 @@ bolt_log.setLevel(logging.ERROR)
PRIMITIVE_NEO4J_TYPES = (integer, string, neo4j_unicode, float, bool)
COMMAND_LAST_RUN_CACHE = CommandLastRunCache()
COURSE_LAST_PUBLISHED_CACHE = CourseLastPublishedCache()
class ModuleStoreSerializer(object):
"""
@@ -45,8 +38,10 @@ class ModuleStoreSerializer(object):
If that parameter isn't furnished, loads all course_keys from the
modulestore.
Filters out course_keys in the `skip` parameter, if provided.
:param courses: string serialization of course keys
:param skip: string serialization of course keys
Args:
courses: A list of string serializations of course keys.
For example, ["course-v1:org+course+run"].
skip: Also a list of string serializations of course keys.
"""
if courses:
course_keys = [CourseKey.from_string(course.strip()) for course in courses]
@@ -67,7 +62,7 @@ class ModuleStoreSerializer(object):
Returns:
fields: a dictionary of an XBlock's field names and values
label: the name of the XBlock's type (i.e. 'course'
block_type: the name of the XBlock's type (i.e. 'course'
or 'problem')
"""
# convert all fields to a dict and filter out parent and children field
@@ -88,25 +83,27 @@ class ModuleStoreSerializer(object):
fields['course_key'] = six.text_type(course_key)
fields['location'] = six.text_type(item.location)
label = item.scope_ids.block_type
block_type = item.scope_ids.block_type
# prune some fields
if label == 'course':
if block_type == 'course':
# prune the checklists field
if 'checklists' in fields:
del fields['checklists']
return fields, label
# record the time this command was run
fields['time_last_dumped_to_neo4j'] = six.text_type(timezone.now())
return fields, block_type
def serialize_course(self, course_id):
"""
Serializes a course into py2neo Nodes and Relationships
Args:
course_id: CourseKey of the course we want to serialize
Returns:
nodes: a list of py2neo Node objects
relationships: a list of py2neo Relationships objects
Serializes a course into Nodes and Relationships
"""
# create a location to node mapping we'll need later for
# writing relationships
@@ -116,12 +113,12 @@ class ModuleStoreSerializer(object):
# create nodes
nodes = []
for item in items:
fields, label = self.serialize_item(item)
fields, block_type = self.serialize_item(item)
for field_name, value in six.iteritems(fields):
fields[field_name] = self.coerce_types(value)
node = Node(label, 'item', **fields)
node = Node(block_type, 'item', **fields)
nodes.append(node)
location_to_node[item.location] = node
@@ -144,7 +141,7 @@ class ModuleStoreSerializer(object):
value: the value of an xblock's field
Returns: either the value, a text version of the value, or, if the
value is a list, a list where each element is converted to text.
value is a list, a list where each element is converted to text.
"""
coerced_value = value
if isinstance(value, list):
@@ -168,44 +165,92 @@ class ModuleStoreSerializer(object):
transaction.create(entity)
@staticmethod
def should_dump_course(course_key):
def get_command_last_run(course_key, graph):
"""
This information is stored on the course node of a course in neo4j
Args:
course_key: a CourseKey
graph: a py2neo Graph
Returns: The datetime that the command was last run, converted into
text, or None, if there's no record of this command last being run.
"""
selector = NodeSelector(graph)
course_node = selector.select(
"course",
course_key=six.text_type(course_key)
).first()
last_this_command_was_run = None
if course_node:
last_this_command_was_run = course_node['time_last_dumped_to_neo4j']
return last_this_command_was_run
@staticmethod
def get_course_last_published(course_key):
"""
We use the CourseStructure table to get when this course was last
published.
Args:
course_key: a CourseKey
Returns: The datetime the course was last published at, converted into
text, or None, if there's no record of the last time this course
was published.
"""
try:
structure = CourseStructure.objects.get(course_id=course_key)
course_last_published_date = six.text_type(structure.modified)
except CourseStructure.DoesNotExist:
course_last_published_date = None
return course_last_published_date
def should_dump_course(self, course_key, graph):
"""
Only dump the course if it's been changed since the last time it's been
dumped.
:param course_key: a CourseKey object.
:return: bool. Whether or not this course should be dumped to neo4j.
Args:
course_key: a CourseKey object.
graph: a py2neo Graph object.
Returns: bool of whether this course should be dumped to neo4j.
"""
last_this_command_was_run = COMMAND_LAST_RUN_CACHE.get(course_key)
last_course_had_published_event = COURSE_LAST_PUBLISHED_CACHE.get(
course_key
)
last_this_command_was_run = self.get_command_last_run(course_key, graph)
# if we have no record of this course being serialized, serialize it
course_last_published_date = self.get_course_last_published(course_key)
# if we don't have a record of the last time this command was run,
# we should serialize the course and dump it
if last_this_command_was_run is None:
return True
# if we've serialized the course recently and we have no published
# events, we can skip re-serializing it
if last_this_command_was_run and last_course_had_published_event is None:
# events, we will not dump it, and so we can skip serializing it
# again here
if last_this_command_was_run and course_last_published_date is None:
return False
# otherwise, serialize if the command was run before the course's last
# published event
return last_this_command_was_run < last_course_had_published_event
# otherwise, serialize and dump the course if the command was run
# before the course's last published event
return last_this_command_was_run < course_last_published_date
def dump_courses_to_neo4j(self, graph, override_cache=False):
"""
Parameters
----------
graph: py2neo graph object
override_cache: serialize the courses even if they'be been recently
serialized
Method that iterates through a list of courses in a modulestore,
serializes them, then writes them to neo4j
Args:
graph: py2neo graph object
override_cache: serialize the courses even if they'be been recently
serialized
Returns two lists: one of the courses that were successfully written
to neo4j, and one of courses that were not.
-------
Returns: two lists--one of the courses that were successfully written
to neo4j and one of courses that were not.
"""
total_number_of_courses = len(self.course_keys)
successful_courses = []
@@ -222,7 +267,7 @@ class ModuleStoreSerializer(object):
total_number_of_courses,
)
if not (override_cache or self.should_dump_course(course_key)):
if not (override_cache or self.should_dump_course(course_key, graph)):
log.info("skipping dumping %s, since it hasn't changed", course_key)
continue
@@ -258,7 +303,6 @@ class ModuleStoreSerializer(object):
unsuccessful_courses.append(course_string)
else:
COMMAND_LAST_RUN_CACHE.set(course_key)
successful_courses.append(course_string)
return successful_courses, unsuccessful_courses

215
openedx/core/djangoapps/coursegraph/management/commands/tests/test_dump_to_neo4j.py
View File

@@ -16,7 +16,13 @@ from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j import (
ModuleStoreSerializer,
)
from openedx.core.djangoapps.coursegraph.signals import _listen_for_course_publish
from openedx.core.djangoapps.coursegraph.management.commands.tests.utils import (
MockGraph,
MockNodeSelector,
)
from openedx.core.djangoapps.content.course_structures.signals import (
listen_for_course_publish
)
class TestDumpToNeo4jCommandBase(SharedModuleStoreTestCase):
@@ -39,6 +45,43 @@ class TestDumpToNeo4jCommandBase(SharedModuleStoreTestCase):
cls.course_strings = [six.text_type(cls.course.id), six.text_type(cls.course2.id)]
@staticmethod
def setup_mock_graph(mock_selector_class, mock_graph_class, transaction_errors=False):
"""
Replaces the py2neo Graph object with a MockGraph; similarly replaces
NodeSelector with MockNodeSelector.
Args:
mock_selector_class: a mocked NodeSelector class
mock_graph_class: a mocked Graph class
transaction_errors: a bool for whether we should get errors
when transactions try to commit
Returns: an instance of MockGraph
"""
mock_graph = MockGraph(transaction_errors=transaction_errors)
mock_graph_class.return_value = mock_graph
mock_node_selector = MockNodeSelector(mock_graph)
mock_selector_class.return_value = mock_node_selector
return mock_graph
def assertCourseDump(self, mock_graph, number_of_courses, number_commits, number_rollbacks):
"""
Asserts that we have the expected number of courses, commits, and
rollbacks after we dump the modulestore to neo4j
Args:
mock_graph: a MockGraph backend
number_of_courses: number of courses we expect to find
number_commits: number of commits we expect against the graph
number_rollbacks: number of commit rollbacks we expect
"""
courses = set([node['course_key'] for node in mock_graph.nodes])
self.assertEqual(len(courses), number_of_courses)
self.assertEqual(mock_graph.number_commits, number_commits)
self.assertEqual(mock_graph.number_rollbacks, number_rollbacks)
@ddt.ddt
class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase):
@@ -46,15 +89,14 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase):
Tests for the dump to neo4j management command
"""
@mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.NodeSelector')
@mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.Graph')
@ddt.data(1, 2)
def test_dump_specific_courses(self, number_of_courses, mock_graph_class):
def test_dump_specific_courses(self, number_of_courses, mock_graph_class, mock_selector_class):
"""
Test that you can specify which courses you want to dump.
"""
mock_graph = mock_graph_class.return_value
mock_transaction = mock.Mock()
mock_graph.begin.return_value = mock_transaction
mock_graph = self.setup_mock_graph(mock_selector_class, mock_graph_class)
call_command(
'dump_to_neo4j',
@@ -65,18 +107,22 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase):
password='mock_password',
)
self.assertEqual(mock_graph.begin.call_count, number_of_courses)
self.assertEqual(mock_transaction.commit.call_count, number_of_courses)
self.assertEqual(mock_transaction.commit.rollback.call_count, 0)
self.assertCourseDump(
mock_graph,
number_of_courses=number_of_courses,
number_commits=number_of_courses,
number_rollbacks=0
)
@mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.NodeSelector')
@mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.Graph')
def test_dump_skip_course(self, mock_graph_class):
def test_dump_skip_course(self, mock_graph_class, mock_selector_class):
"""
Test that you can skip courses.
"""
mock_graph = mock_graph_class.return_value
mock_transaction = mock.Mock()
mock_graph.begin.return_value = mock_transaction
mock_graph = self.setup_mock_graph(
mock_selector_class, mock_graph_class
)
call_command(
'dump_to_neo4j',
@@ -87,18 +133,22 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase):
password='mock_password',
)
self.assertEqual(mock_graph.begin.call_count, 1)
self.assertEqual(mock_transaction.commit.call_count, 1)
self.assertEqual(mock_transaction.commit.rollback.call_count, 0)
self.assertCourseDump(
mock_graph,
number_of_courses=1,
number_commits=1,
number_rollbacks=0,
)
@mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.NodeSelector')
@mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.Graph')
def test_dump_skip_beats_specifying(self, mock_graph_class):
def test_dump_skip_beats_specifying(self, mock_graph_class, mock_selector_class):
"""
Test that if you skip and specify the same course, you'll skip it.
"""
mock_graph = mock_graph_class.return_value
mock_transaction = mock.Mock()
mock_graph.begin.return_value = mock_transaction
mock_graph = self.setup_mock_graph(
mock_selector_class, mock_graph_class
)
call_command(
'dump_to_neo4j',
@@ -110,31 +160,38 @@ class TestDumpToNeo4jCommand(TestDumpToNeo4jCommandBase):
password='mock_password',
)
self.assertEqual(mock_graph.begin.call_count, 0)
self.assertEqual(mock_transaction.commit.call_count, 0)
self.assertEqual(mock_transaction.commit.rollback.call_count, 0)
self.assertCourseDump(
mock_graph,
number_of_courses=0,
number_commits=0,
number_rollbacks=0,
)
@mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.NodeSelector')
@mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.Graph')
def test_dump_all_courses(self, mock_graph_class):
def test_dump_all_courses(self, mock_graph_class, mock_selector_class):
"""
Test if you don't specify which courses to dump, then you'll dump
all of them.
"""
mock_graph = mock_graph_class.return_value
mock_transaction = mock.Mock()
mock_graph.begin.return_value = mock_transaction
mock_graph = self.setup_mock_graph(
mock_selector_class, mock_graph_class
)
call_command(
'dump_to_neo4j',
host='mock_host',
http_port=7474,
user='mock_user',
password='mock_password',
password='mock_password'
)
self.assertEqual(mock_graph.begin.call_count, 2)
self.assertEqual(mock_transaction.commit.call_count, 2)
self.assertEqual(mock_transaction.commit.rollback.call_count, 0)
self.assertCourseDump(
mock_graph,
number_of_courses=2,
number_commits=2,
number_rollbacks=0,
)
@ddt.ddt
@@ -167,9 +224,7 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase):
"""
Tests the serialize_course method.
"""
nodes, relationships = self.mss.serialize_course(
self.course.id
)
nodes, relationships = self.mss.serialize_course(self.course.id)
self.assertEqual(len(nodes), 9)
self.assertEqual(len(relationships), 7)
@@ -194,63 +249,68 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase):
coerced_value = self.mss.coerce_types(original_value)
self.assertEqual(coerced_value, coerced_expected)
def test_dump_to_neo4j(self):
@mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.NodeSelector')
def test_dump_to_neo4j(self, mock_selector_class):
"""
Tests the dump_to_neo4j method works against a mock
py2neo Graph
"""
mock_graph = mock.Mock()
mock_transaction = mock.Mock()
mock_graph.begin.return_value = mock_transaction
mock_graph = MockGraph()
mock_selector_class.return_value = MockNodeSelector(mock_graph)
successful, unsuccessful = self.mss.dump_courses_to_neo4j(mock_graph)
self.assertEqual(mock_graph.begin.call_count, 2)
self.assertEqual(mock_transaction.commit.call_count, 2)
self.assertEqual(mock_transaction.rollback.call_count, 0)
self.assertCourseDump(
mock_graph,
number_of_courses=2,
number_commits=2,
number_rollbacks=0,
)
# 7 nodes + 9 relationships from the first course
# 9 nodes + 7 relationships from the first course
# 2 nodes and no relationships from the second
self.assertEqual(mock_transaction.create.call_count, 18)
self.assertEqual(mock_transaction.run.call_count, 2)
self.assertEqual(len(mock_graph.nodes), 11)
self.assertEqual(len(unsuccessful), 0)
self.assertItemsEqual(successful, self.course_strings)
def test_dump_to_neo4j_rollback(self):
@mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.NodeSelector')
def test_dump_to_neo4j_rollback(self, mock_selector_class):
"""
Tests that the the dump_to_neo4j method handles the case where there's
an exception trying to write to the neo4j database.
"""
mock_graph = mock.Mock()
mock_transaction = mock.Mock()
mock_graph.begin.return_value = mock_transaction
mock_transaction.run.side_effect = ValueError('Something went wrong!')
mock_graph = MockGraph(transaction_errors=True)
mock_selector_class.return_value = MockNodeSelector(mock_graph)
successful, unsuccessful = self.mss.dump_courses_to_neo4j(mock_graph)
self.assertEqual(mock_graph.begin.call_count, 2)
self.assertEqual(mock_transaction.commit.call_count, 0)
self.assertEqual(mock_transaction.rollback.call_count, 2)
self.assertCourseDump(
mock_graph,
number_of_courses=0,
number_commits=0,
number_rollbacks=2,
)
self.assertEqual(len(successful), 0)
self.assertItemsEqual(unsuccessful, self.course_strings)
@ddt.data(
(True, 2),
(False, 0),
)
@mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.NodeSelector')
@ddt.data((True, 2), (False, 0))
@ddt.unpack
def test_dump_to_neo4j_cache(self, override_cache, expected_number_courses):
def test_dump_to_neo4j_cache(self, override_cache, expected_number_courses, mock_selector_class):
"""
Tests the caching mechanism and override to make sure we only publish
recently updated courses.
"""
mock_graph = mock.Mock()
mock_graph = MockGraph()
mock_selector_class.return_value = MockNodeSelector(mock_graph)
# run once to warm the cache
successful, unsuccessful = self.mss.dump_courses_to_neo4j(mock_graph)
self.assertEqual(len(successful + unsuccessful), len(self.course_strings))
self.mss.dump_courses_to_neo4j(
mock_graph, override_cache=override_cache
)
# when run the second time, only dump courses if the cache override
# is enabled
@@ -259,19 +319,21 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase):
)
self.assertEqual(len(successful + unsuccessful), expected_number_courses)
def test_dump_to_neo4j_published(self):
@mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.NodeSelector')
def test_dump_to_neo4j_published(self, mock_selector_class):
"""
Tests that we only dump those courses that have been published after
the last time the command was been run.
"""
mock_graph = mock.Mock()
mock_graph = MockGraph()
mock_selector_class.return_value = MockNodeSelector(mock_graph)
# run once to warm the cache
successful, unsuccessful = self.mss.dump_courses_to_neo4j(mock_graph)
self.assertEqual(len(successful + unsuccessful), len(self.course_strings))
# simulate one of the courses being published
_listen_for_course_publish(None, self.course.id)
listen_for_course_publish(None, self.course.id)
# make sure only the published course was dumped
successful, unsuccessful = self.mss.dump_courses_to_neo4j(mock_graph)
@@ -280,31 +342,24 @@ class TestModuleStoreSerializer(TestDumpToNeo4jCommandBase):
self.assertEqual(successful[0], unicode(self.course.id))
@ddt.data(
(datetime(2016, 3, 30), datetime(2016, 3, 31), True),
(datetime(2016, 3, 31), datetime(2016, 3, 30), False),
(datetime(2016, 3, 31), None, False),
(None, datetime(2016, 3, 30), True),
(six.text_type(datetime(2016, 3, 30)), six.text_type(datetime(2016, 3, 31)), True),
(six.text_type(datetime(2016, 3, 31)), six.text_type(datetime(2016, 3, 30)), False),
(six.text_type(datetime(2016, 3, 31)), None, False),
(None, six.text_type(datetime(2016, 3, 30)), True),
(None, None, True),
)
@ddt.unpack
@mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.COMMAND_LAST_RUN_CACHE')
@mock.patch('openedx.core.djangoapps.coursegraph.management.commands.dump_to_neo4j.COURSE_LAST_PUBLISHED_CACHE')
def test_should_dump_course(
self,
last_command_run,
last_course_published,
should_dump,
mock_course_last_published_cache,
mock_command_last_run_cache,
):
def test_should_dump_course(self, last_command_run, last_course_published, should_dump):
"""
Tests whether a course should be dumped given the last time it was
dumped and the last time it was published.
"""
mock_command_last_run_cache.get.return_value = last_command_run
mock_course_last_published_cache.get.return_value = last_course_published
mss = ModuleStoreSerializer()
mss.get_command_last_run = lambda course_key, graph: last_command_run
mss.get_course_last_published = lambda course_key: last_course_published
mock_course_key = mock.Mock
mock_graph = mock.Mock()
self.assertEqual(
self.mss.should_dump_course(mock_course_key),
should_dump
mss.should_dump_course(mock_course_key, mock_graph),
should_dump,
)

123
openedx/core/djangoapps/coursegraph/management/commands/tests/utils.py Normal file
View File

@@ -0,0 +1,123 @@
"""
Utilities for testing the dump_to_neo4j management command
"""
from __future__ import unicode_literals
from py2neo import Node
class MockGraph(object):
"""
A stubbed out version of py2neo's Graph object, used for testing.
Args:
transaction_errors: a bool for whether transactions should throw
an error.
"""
def __init__(self, transaction_errors=False, **kwargs): # pylint: disable=unused-argument
self.nodes = set()
self.number_commits = 0
self.number_rollbacks = 0
self.transaction_errors = transaction_errors
def begin(self):
"""
A stub of the method that generates transactions
Returns: a MockTransaction object (instead of a py2neo Transaction)
"""
return MockTransaction(self)
class MockTransaction(object):
"""
A stubbed out version of py2neo's Transaction object, used for testing.
"""
def __init__(self, graph):
self.temp = set()
self.graph = graph
def run(self, query):
"""
Deletes all nodes associated with a course. Normally `run` executes
an arbitrary query, but in our code, we only use it to delete nodes
associated with a course.
Args:
query: query string to be executed (in this case, to delete all
nodes associated with a course)
"""
start_string = "WHERE n.course_key='"
start = query.index(start_string) + len(start_string)
query = query[start:]
end = query.find("'")
course_key = query[:end]
self.graph.nodes = set([
node for node in self.graph.nodes if node['course_key'] != course_key
])
def create(self, element):
"""
Adds elements to the transaction's temporary backend storage
Args:
element: a py2neo Node object
"""
if isinstance(element, Node):
self.temp.add(element)
def commit(self):
"""
Takes elements in the transaction's temporary storage and adds them
to the mock graph's storage. Throws an error if the graph's
transaction_errors param is set to True.
"""
if self.graph.transaction_errors:
raise Exception("fake exception while trying to commit")
for element in self.temp:
self.graph.nodes.add(element)
self.temp.clear()
self.graph.number_commits += 1
def rollback(self):
"""
Clears the transactions temporary storage
"""
self.temp.clear()
self.graph.number_rollbacks += 1
class MockNodeSelector(object):
"""
Mocks out py2neo's NodeSelector class. Used to select a node from a graph.
py2neo's NodeSelector expects a real graph object to run queries against,
so, rather than have to mock out MockGraph to accommodate those queries,
it seemed simpler to mock out NodeSelector as well.
"""
def __init__(self, graph):
self.graph = graph
def select(self, label, course_key):
"""
Selects nodes that match a label and course_key
Args:
label: the string of the label we're selecting nodes by
course_key: the string of the course key we're selecting node by
Returns: a MockResult of matching nodes
"""
nodes = []
for node in self.graph.nodes:
if node.has_label(label) and node["course_key"] == course_key:
nodes.append(node)
return MockNodeSelection(nodes)
class MockNodeSelection(list):
"""
Mocks out py2neo's NodeSelection class: this is the type of what
MockNodeSelector's `select` method returns.
"""
def first(self):
"""
Returns: the first element of a list if the list has elements.
Otherwise, None.
"""
return self[0] if self else None

15
openedx/core/djangoapps/coursegraph/signals.py
View File

@@ -1,15 +0,0 @@
"""
Signal handlers for the CourseGraph application
"""
from django.dispatch.dispatcher import receiver
from xmodule.modulestore.django import SignalHandler
from openedx.core.djangoapps.coursegraph.utils import CourseLastPublishedCache
@receiver(SignalHandler.course_published)
def _listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable=unused-argument
"""
Register when the course was published on a course publish event
"""
CourseLastPublishedCache().set(course_key)

0
openedx/core/djangoapps/coursegraph/tests/__init__.py
View File

27
openedx/core/djangoapps/coursegraph/tests/test_signals.py
View File

@@ -1,27 +0,0 @@
"""
Tests for coursegraph's signal handler on course publish
"""
from __future__ import unicode_literals
from opaque_keys.edx.keys import CourseKey
from openedx.core.djangoapps.coursegraph.signals import _listen_for_course_publish
from openedx.core.djangoapps.coursegraph.utils import CourseLastPublishedCache
from openedx.core.djangolib.testing.utils import CacheIsolationTestCase
class TestCourseGraphSignalHandler(CacheIsolationTestCase):
"""
Tests for the course publish course handler
"""
ENABLED_CACHES = ['default']
def test_cache_set_on_course_publish(self):
"""
Tests that the last published cache is set on course publish
"""
course_key = CourseKey.from_string('course-v1:org+course+run')
last_published_cache = CourseLastPublishedCache()
self.assertIsNone(last_published_cache.get(course_key))
_listen_for_course_publish(None, course_key)
self.assertIsNotNone(last_published_cache.get(course_key))

52
openedx/core/djangoapps/coursegraph/utils.py
View File

@@ -1,52 +0,0 @@
"""
Helpers for the CourseGraph app
"""
from django.core.cache import cache
from django.utils import timezone
class TimeRecordingCacheBase(object):
"""
A base class for caching the current time for some key.
"""
# cache_prefix should be defined in children classes
cache_prefix = None
_cache = cache
def _key(self, course_key):
"""
Make a cache key from the prefix and a course_key
:param course_key: CourseKey object
:return: a cache key
"""
return self.cache_prefix + unicode(course_key)
def get(self, course_key):
"""
Gets the time value associated with the CourseKey.
:param course_key: a CourseKey object.
:return: the time the key was last set.
"""
return self._cache.get(self._key(course_key))
def set(self, course_key):
"""
Sets the current time for a CourseKey key.
:param course_key: a CourseKey object.
"""
return self._cache.set(self._key(course_key), timezone.now())
class CourseLastPublishedCache(TimeRecordingCacheBase):
"""
Used to record the last time that a course had a publish event run on it.
"""
cache_prefix = u'course_last_published'
class CommandLastRunCache(TimeRecordingCacheBase):
"""
Used to record the last time that the dump_to_neo4j command was run on a
course.
"""
cache_prefix = u'dump_to_neo4j_command_last_run'