Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add ConfigurationModel deserializer and management command.

Browse Source 
TNL-4781, TNL-4782
This commit is contained in:
cahrens
2016-06-10 16:36:23 -04:00
parent 9abce102ff
commit 0a6406e863
10 changed files with 556 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
common/djangoapps/config_models/management/__init__.py Normal file
View File

0
common/djangoapps/config_models/management/commands/__init__.py Normal file
View File

72
common/djangoapps/config_models/management/commands/populate_model.py Normal file
View File

@@ -0,0 +1,72 @@
"""
Populates a ConfigurationModel by deserializing JSON data contained in a file.
"""
import os
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from django.utils.translation import ugettext_lazy as _
from config_models.utils import deserialize_json
class Command(BaseCommand):
"""
This command will deserialize the JSON data in the supplied file to populate
a ConfigurationModel. Note that this will add new entries to the model, but it
will not delete any entries (ConfigurationModel entries are read-only).
"""
help = """
Populates a ConfigurationModel by deserializing the supplied JSON.
JSON should be in a file, with the following format:
{ "model": "config_models.ExampleConfigurationModel",
"data":
[
{ "enabled": True,
"color": "black"
...
},
{ "enabled": False,
"color": "yellow"
...
},
...
]
}
A username corresponding to an existing user must be specified to indicate who
is executing the command.
$ ... populate_model -f path/to/file.json -u username
"""
option_list = BaseCommand.option_list + (
make_option('-f', '--file',
metavar='JSON_FILE',
dest='file',
default=False,
help='JSON file to import ConfigurationModel data'),
make_option('-u', '--username',
metavar='USERNAME',
dest='username',
default=False,
help='username to specify who is executing the command'),
)
def handle(self, *args, **options):
if 'file' not in options or not options['file']:
raise CommandError(_("A file containing JSON must be specified."))
if 'username' not in options or not options['username']:
raise CommandError(_("A valid username must be specified."))
json_file = options['file']
if not os.path.exists(json_file):
raise CommandError(_("File {0} does not exist").format(json_file))
self.stdout.write(_("Importing JSON data from file {0}").format(json_file))
with open(json_file) as data:
created_entries = deserialize_json(data, options['username'])
self.stdout.write(_("Import complete, {0} new entries created").format(created_entries))

58
common/djangoapps/config_models/models.py
View File

@@ -6,6 +6,9 @@ from django.contrib.auth.models import User
from django.core.cache import caches, InvalidCacheBackendError
from django.utils.translation import ugettext_lazy as _
from rest_framework.utils import model_meta
try:
cache = caches['configuration'] # pylint: disable=invalid-name
except InvalidCacheBackendError:
@@ -176,3 +179,58 @@ class ConfigurationModel(models.Model):
values = list(cls.objects.values_list(*key_fields, flat=flat).order_by().distinct())
cache.set(cache_key, values, cls.cache_timeout)
return values
def fields_equal(self, instance, fields_to_ignore=("id", "change_date", "changed_by")):
"""
Compares this instance's fields to the supplied instance to test for equality.
This will ignore any fields in `fields_to_ignore`.
Note that this method ignores many-to-many fields.
Args:
instance: the model instance to compare
fields_to_ignore: List of fields that should not be compared for equality. By default
includes `id`, `change_date`, and `changed_by`.
Returns: True if the checked fields are all equivalent, else False
"""
for field in self._meta.get_fields():
if not field.many_to_many and field.name not in fields_to_ignore:
if getattr(instance, field.name) != getattr(self, field.name):
return False
return True
@classmethod
def equal_to_current(cls, json, fields_to_ignore=("id", "change_date", "changed_by")):
"""
Compares for equality this instance to a model instance constructed from the supplied JSON.
This will ignore any fields in `fields_to_ignore`.
Note that this method cannot handle fields with many-to-many associations, as those can only
be set on a saved model instance (and saving the model instance will create a new entry).
All many-to-many field entries will be removed before the equality comparison is done.
Args:
json: json representing an entry to compare
fields_to_ignore: List of fields that should not be compared for equality. By default
includes `id`, `change_date`, and `changed_by`.
Returns: True if the checked fields are all equivalent, else False
"""
# Remove many-to-many relationships from json.
# They require an instance to be already saved.
info = model_meta.get_field_info(cls)
for field_name, relation_info in info.relations.items():
if relation_info.to_many and (field_name in json):
json.pop(field_name)
new_instance = cls(**json)
key_field_args = tuple(getattr(new_instance, key) for key in cls.KEY_FIELDS)
current = cls.current(*key_field_args)
# If current.id is None, no entry actually existed and the "current" method created it.
if current.id is not None:
return current.fields_equal(new_instance, fields_to_ignore)
return False

0
common/djangoapps/config_models/tests/__init__.py Normal file
View File

14
common/djangoapps/config_models/tests/data/data.json Normal file
View File

@@ -0,0 +1,14 @@
{
"model": "config_models.ExampleDeserializeConfig",
"data": [
{
"name": "betty",
"enabled": true,
"int_field": 5
},
{
"name": "fred",
"enabled": false
}
]
}

219
common/djangoapps/config_models/tests/test_model_deserialization.py Normal file
View File

@@ -0,0 +1,219 @@
"""
Tests of the populate_model management command and its helper utils.deserialize_json method.
"""
import textwrap
import os.path
from django.utils import timezone
from django.utils.six import BytesIO
from django.contrib.auth.models import User
from django.core.management.base import CommandError
from django.db import models
from config_models.management.commands import populate_model
from config_models.models import ConfigurationModel
from config_models.utils import deserialize_json
from openedx.core.djangolib.testing.utils import CacheIsolationTestCase
class ExampleDeserializeConfig(ConfigurationModel):
"""
Test model for testing deserialization of ``ConfigurationModels`` with keyed configuration.
"""
KEY_FIELDS = ('name',)
name = models.TextField()
int_field = models.IntegerField(default=10)
def __unicode__(self):
return "ExampleDeserializeConfig(enabled={}, name={}, int_field={})".format(
self.enabled, self.name, self.int_field
)
class DeserializeJSONTests(CacheIsolationTestCase):
"""
Tests of deserializing the JSON representation of ConfigurationModels.
"""
def setUp(self):
super(DeserializeJSONTests, self).setUp()
self.test_username = 'test_worker'
User.objects.create_user(username=self.test_username)
self.fixture_path = os.path.join(os.path.dirname(__file__), 'data', 'data.json')
def test_deserialize_models(self):
"""
Tests the "happy path", where 2 instances of the test model should be created.
A valid username is supplied for the operation.
"""
start_date = timezone.now()
with open(self.fixture_path) as data:
entries_created = deserialize_json(data, self.test_username)
self.assertEquals(2, entries_created)
self.assertEquals(2, ExampleDeserializeConfig.objects.count())
betty = ExampleDeserializeConfig.current('betty')
self.assertTrue(betty.enabled)
self.assertEquals(5, betty.int_field)
self.assertGreater(betty.change_date, start_date)
self.assertEquals(self.test_username, betty.changed_by.username)
fred = ExampleDeserializeConfig.current('fred')
self.assertFalse(fred.enabled)
self.assertEquals(10, fred.int_field)
self.assertGreater(fred.change_date, start_date)
self.assertEquals(self.test_username, fred.changed_by.username)
def test_existing_entries_not_removed(self):
"""
Any existing configuration model entries are retained
(though they may be come history)-- deserialize_json is purely additive.
"""
ExampleDeserializeConfig(name="fred", enabled=True).save()
ExampleDeserializeConfig(name="barney", int_field=200).save()
with open(self.fixture_path) as data:
entries_created = deserialize_json(data, self.test_username)
self.assertEquals(2, entries_created)
self.assertEquals(4, ExampleDeserializeConfig.objects.count())
self.assertEquals(3, len(ExampleDeserializeConfig.objects.current_set()))
self.assertEquals(5, ExampleDeserializeConfig.current('betty').int_field)
self.assertEquals(200, ExampleDeserializeConfig.current('barney').int_field)
# The JSON file changes "enabled" to False for Fred.
fred = ExampleDeserializeConfig.current('fred')
self.assertFalse(fred.enabled)
def test_duplicate_entries_not_made(self):
"""
If there is no change in an entry (besides changed_by and change_date),
a new entry is not made.
"""
with open(self.fixture_path) as data:
entries_created = deserialize_json(data, self.test_username)
self.assertEquals(2, entries_created)
with open(self.fixture_path) as data:
entries_created = deserialize_json(data, self.test_username)
self.assertEquals(0, entries_created)
# Importing twice will still only result in 2 records (second import a no-op).
self.assertEquals(2, ExampleDeserializeConfig.objects.count())
# Change Betty.
betty = ExampleDeserializeConfig.current('betty')
betty.int_field = -8
betty.save()
self.assertEquals(3, ExampleDeserializeConfig.objects.count())
self.assertEquals(-8, ExampleDeserializeConfig.current('betty').int_field)
# Now importing will add a new entry for Betty.
with open(self.fixture_path) as data:
entries_created = deserialize_json(data, self.test_username)
self.assertEquals(1, entries_created)
self.assertEquals(4, ExampleDeserializeConfig.objects.count())
self.assertEquals(5, ExampleDeserializeConfig.current('betty').int_field)
def test_bad_username(self):
"""
Tests the error handling when the specified user does not exist.
"""
test_json = textwrap.dedent("""
{
"model": "config_models.ExampleDeserializeConfig",
"data": [{"name": "dino"}]
}
""")
with self.assertRaisesRegexp(Exception, "User matching query does not exist"):
deserialize_json(BytesIO(test_json), "unknown_username")
def test_invalid_json(self):
"""
Tests the error handling when there is invalid JSON.
"""
test_json = textwrap.dedent("""
{
"model": "config_models.ExampleDeserializeConfig",
"data": [{"name": "dino"
""")
with self.assertRaisesRegexp(Exception, "JSON parse error"):
deserialize_json(BytesIO(test_json), self.test_username)
def test_invalid_model(self):
"""
Tests the error handling when the configuration model specified does not exist.
"""
test_json = textwrap.dedent("""
{
"model": "xxx.yyy",
"data":[{"name": "dino"}]
}
""")
with self.assertRaisesRegexp(Exception, "No installed app"):
deserialize_json(BytesIO(test_json), self.test_username)
class PopulateModelTestCase(CacheIsolationTestCase):
"""
Tests of populate model management command.
"""
def setUp(self):
super(PopulateModelTestCase, self).setUp()
self.file_path = os.path.join(os.path.dirname(__file__), 'data', 'data.json')
self.test_username = 'test_management_worker'
User.objects.create_user(username=self.test_username)
def test_run_command(self):
"""
Tests the "happy path", where 2 instances of the test model should be created.
A valid username is supplied for the operation.
"""
_run_command(file=self.file_path, username=self.test_username)
self.assertEquals(2, ExampleDeserializeConfig.objects.count())
betty = ExampleDeserializeConfig.current('betty')
self.assertEquals(self.test_username, betty.changed_by.username)
fred = ExampleDeserializeConfig.current('fred')
self.assertEquals(self.test_username, fred.changed_by.username)
def test_no_user_specified(self):
"""
Tests that a username must be specified.
"""
with self.assertRaisesRegexp(CommandError, "A valid username must be specified"):
_run_command(file=self.file_path)
def test_bad_user_specified(self):
"""
Tests that a username must be specified.
"""
with self.assertRaisesRegexp(Exception, "User matching query does not exist"):
_run_command(file=self.file_path, username="does_not_exist")
def test_no_file_specified(self):
"""
Tests the error handling when no JSON file is supplied.
"""
with self.assertRaisesRegexp(CommandError, "A file containing JSON must be specified"):
_run_command(username=self.test_username)
def test_bad_file_specified(self):
"""
Tests the error handling when the path to the JSON file is incorrect.
"""
with self.assertRaisesRegexp(CommandError, "File does/not/exist.json does not exist"):
_run_command(file="does/not/exist.json", username=self.test_username)
def _run_command(*args, **kwargs):
"""Run the management command to deserializer JSON ConfigurationModel data. """
command = populate_model.Command()
return command.handle(*args, **kwargs)

122
common/djangoapps/config_models/tests.py → common/djangoapps/config_models/tests/tests.py
View File

@@ -25,6 +25,24 @@ class ExampleConfig(ConfigurationModel):
string_field = models.TextField()
int_field = models.IntegerField(default=10)
def __unicode__(self):
return "ExampleConfig(enabled={}, string_field={}, int_field={})".format(
self.enabled, self.string_field, self.int_field
)
class ManyToManyExampleConfig(ConfigurationModel):
"""
Test model configuration with a many-to-many field.
"""
cache_timeout = 300
string_field = models.TextField()
many_user_field = models.ManyToManyField(User, related_name='topic_many_user_field')
def __unicode__(self):
return "ManyToManyExampleConfig(enabled={}, string_field={})".format(self.enabled, self.string_field)
@patch('config_models.models.cache')
class ConfigurationModelTests(TestCase):
@@ -40,7 +58,7 @@ class ConfigurationModelTests(TestCase):
ExampleConfig(changed_by=self.user).save()
mock_cache.delete.assert_called_with(ExampleConfig.cache_key_name())
def test_cache_key_name(self, _mock_cache):
def test_cache_key_name(self, __):
self.assertEquals(ExampleConfig.cache_key_name(), 'configuration/ExampleConfig/current')
def test_no_config_empty_cache(self, mock_cache):
@@ -103,6 +121,64 @@ class ConfigurationModelTests(TestCase):
self.assertEquals(2, ExampleConfig.objects.all().count())
def test_equality(self, mock_cache):
mock_cache.get.return_value = None
config = ExampleConfig(changed_by=self.user, string_field='first')
config.save()
self.assertTrue(ExampleConfig.equal_to_current({"string_field": "first"}))
self.assertTrue(ExampleConfig.equal_to_current({"string_field": "first", "enabled": False}))
self.assertTrue(ExampleConfig.equal_to_current({"string_field": "first", "int_field": 10}))
self.assertFalse(ExampleConfig.equal_to_current({"string_field": "first", "enabled": True}))
self.assertFalse(ExampleConfig.equal_to_current({"string_field": "first", "int_field": 20}))
self.assertFalse(ExampleConfig.equal_to_current({"string_field": "second"}))
self.assertFalse(ExampleConfig.equal_to_current({}))
def test_equality_custom_fields_to_ignore(self, mock_cache):
mock_cache.get.return_value = None
config = ExampleConfig(changed_by=self.user, string_field='first')
config.save()
# id, change_date, and changed_by will all be different for a newly created entry
self.assertTrue(ExampleConfig.equal_to_current({"string_field": "first"}))
self.assertFalse(
ExampleConfig.equal_to_current({"string_field": "first"}, fields_to_ignore=("change_date", "changed_by"))
)
self.assertFalse(
ExampleConfig.equal_to_current({"string_field": "first"}, fields_to_ignore=("id", "changed_by"))
)
self.assertFalse(
ExampleConfig.equal_to_current({"string_field": "first"}, fields_to_ignore=("change_date", "id"))
)
# Test the ability to ignore a different field ("int_field").
self.assertFalse(ExampleConfig.equal_to_current({"string_field": "first", "int_field": 20}))
self.assertTrue(
ExampleConfig.equal_to_current(
{"string_field": "first", "int_field": 20},
fields_to_ignore=("id", "change_date", "changed_by", "int_field")
)
)
def test_equality_ignores_many_to_many(self, mock_cache):
mock_cache.get.return_value = None
config = ManyToManyExampleConfig(changed_by=self.user, string_field='first')
config.save()
second_user = User(username="second_user")
second_user.save()
config.many_user_field.add(second_user) # pylint: disable=no-member
config.save()
# The many-to-many field is ignored in comparison.
self.assertTrue(
ManyToManyExampleConfig.equal_to_current({"string_field": "first", "many_user_field": "removed"})
)
class ExampleKeyedConfig(ConfigurationModel):
"""
@@ -120,6 +196,11 @@ class ExampleKeyedConfig(ConfigurationModel):
string_field = models.TextField()
int_field = models.IntegerField(default=10)
def __unicode__(self):
return "ExampleKeyedConfig(enabled={}, left={}, right={}, string_field={}, int_field={})".format(
self.enabled, self.left, self.right, self.string_field, self.int_field
)
@ddt.ddt
@patch('config_models.models.cache')
@@ -294,6 +375,45 @@ class KeyedConfigurationModelTests(TestCase):
mock_cache.get.return_value = fake_result
self.assertEquals(ExampleKeyedConfig.key_values(), fake_result)
def test_equality(self, mock_cache):
mock_cache.get.return_value = None
config1 = ExampleKeyedConfig(left='left_a', right='right_a', int_field=1, changed_by=self.user)
config1.save()
config2 = ExampleKeyedConfig(left='left_b', right='right_b', int_field=2, changed_by=self.user, enabled=True)
config2.save()
config3 = ExampleKeyedConfig(left='left_c', changed_by=self.user)
config3.save()
self.assertTrue(
ExampleKeyedConfig.equal_to_current({"left": "left_a", "right": "right_a", "int_field": 1})
)
self.assertTrue(
ExampleKeyedConfig.equal_to_current({"left": "left_b", "right": "right_b", "int_field": 2, "enabled": True})
)
self.assertTrue(
ExampleKeyedConfig.equal_to_current({"left": "left_c"})
)
self.assertFalse(
ExampleKeyedConfig.equal_to_current(
{"left": "left_a", "right": "right_a", "int_field": 1, "string_field": "foo"}
)
)
self.assertFalse(
ExampleKeyedConfig.equal_to_current({"left": "left_a", "int_field": 1})
)
self.assertFalse(
ExampleKeyedConfig.equal_to_current({"left": "left_b", "right": "right_b", "int_field": 2})
)
self.assertFalse(
ExampleKeyedConfig.equal_to_current({"left": "left_c", "int_field": 11})
)
self.assertFalse(ExampleKeyedConfig.equal_to_current({}))
@ddt.ddt
class ConfigurationModelAPITests(TestCase):

69
common/djangoapps/config_models/utils.py Normal file
View File

@@ -0,0 +1,69 @@
"""
Utilities for working with ConfigurationModels.
"""
from django.apps import apps
from rest_framework.parsers import JSONParser
from rest_framework.serializers import ModelSerializer
from django.contrib.auth.models import User
def get_serializer_class(configuration_model):
""" Returns a ConfigurationModel serializer class for the supplied configuration_model. """
class AutoConfigModelSerializer(ModelSerializer):
"""Serializer class for configuration models."""
class Meta(object):
"""Meta information for AutoConfigModelSerializer."""
model = configuration_model
def create(self, validated_data):
if "changed_by_username" in self.context:
validated_data['changed_by'] = User.objects.get(username=self.context["changed_by_username"])
return super(AutoConfigModelSerializer, self).create(validated_data)
return AutoConfigModelSerializer
def deserialize_json(stream, username):
"""
Given a stream containing JSON, deserializers the JSON into ConfigurationModel instances.
The stream is expected to be in the following format:
{ "model": "config_models.ExampleConfigurationModel",
"data":
[
{ "enabled": True,
"color": "black"
...
},
{ "enabled": False,
"color": "yellow"
...
},
...
]
}
If the provided stream does not contain valid JSON for the ConfigurationModel specified,
an Exception will be raised.
Arguments:
stream: The stream of JSON, as described above.
username: The username of the user making the change. This must match an existing user.
Returns: the number of created entries
"""
parsed_json = JSONParser().parse(stream)
serializer_class = get_serializer_class(apps.get_model(parsed_json["model"]))
list_serializer = serializer_class(data=parsed_json["data"], context={"changed_by_username": username}, many=True)
if list_serializer.is_valid():
model_class = serializer_class.Meta.model
for data in reversed(list_serializer.validated_data):
if model_class.equal_to_current(data):
list_serializer.validated_data.remove(data)
entries_created = len(list_serializer.validated_data)
list_serializer.save()
return entries_created
else:
raise Exception(list_serializer.error_messages)

11
common/djangoapps/config_models/views.py
View File

@@ -4,9 +4,10 @@ API view to allow manipulation of configuration models.
from rest_framework.generics import CreateAPIView, RetrieveAPIView
from rest_framework.permissions import DjangoModelPermissions
from rest_framework.authentication import SessionAuthentication
from rest_framework.serializers import ModelSerializer
from django.db import transaction
from config_models.utils import get_serializer_class
class ReadableOnlyByAuthors(DjangoModelPermissions):
"""Only allow access by users with `add` permissions on the model."""
@@ -58,13 +59,7 @@ class ConfigurationModelCurrentAPIView(AtomicMixin, CreateAPIView, RetrieveAPIVi
def get_serializer_class(self):
if self.serializer_class is None:
class AutoConfigModelSerializer(ModelSerializer):
"""Serializer class for configuration models."""
class Meta(object):
"""Meta information for AutoConfigModelSerializer."""
model = self.model
self.serializer_class = AutoConfigModelSerializer
self.serializer_class = get_serializer_class(self.model)
return self.serializer_class