diff --git a/common/djangoapps/config_models/management/__init__.py b/common/djangoapps/config_models/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/common/djangoapps/config_models/management/commands/__init__.py b/common/djangoapps/config_models/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/common/djangoapps/config_models/management/commands/populate_model.py b/common/djangoapps/config_models/management/commands/populate_model.py new file mode 100644 index 0000000000..d41ceae58b --- /dev/null +++ b/common/djangoapps/config_models/management/commands/populate_model.py @@ -0,0 +1,72 @@ +""" +Populates a ConfigurationModel by deserializing JSON data contained in a file. +""" +import os +from optparse import make_option + +from django.core.management.base import BaseCommand, CommandError +from django.utils.translation import ugettext_lazy as _ + +from config_models.utils import deserialize_json + + +class Command(BaseCommand): + """ + This command will deserialize the JSON data in the supplied file to populate + a ConfigurationModel. Note that this will add new entries to the model, but it + will not delete any entries (ConfigurationModel entries are read-only). + """ + help = """ + Populates a ConfigurationModel by deserializing the supplied JSON. + + JSON should be in a file, with the following format: + + { "model": "config_models.ExampleConfigurationModel", + "data": + [ + { "enabled": True, + "color": "black" + ... + }, + { "enabled": False, + "color": "yellow" + ... + }, + ... + ] + } + + A username corresponding to an existing user must be specified to indicate who + is executing the command. + + $ ... populate_model -f path/to/file.json -u username + """ + + option_list = BaseCommand.option_list + ( + make_option('-f', '--file', + metavar='JSON_FILE', + dest='file', + default=False, + help='JSON file to import ConfigurationModel data'), + make_option('-u', '--username', + metavar='USERNAME', + dest='username', + default=False, + help='username to specify who is executing the command'), + ) + + def handle(self, *args, **options): + if 'file' not in options or not options['file']: + raise CommandError(_("A file containing JSON must be specified.")) + + if 'username' not in options or not options['username']: + raise CommandError(_("A valid username must be specified.")) + + json_file = options['file'] + if not os.path.exists(json_file): + raise CommandError(_("File {0} does not exist").format(json_file)) + + self.stdout.write(_("Importing JSON data from file {0}").format(json_file)) + with open(json_file) as data: + created_entries = deserialize_json(data, options['username']) + self.stdout.write(_("Import complete, {0} new entries created").format(created_entries)) diff --git a/common/djangoapps/config_models/models.py b/common/djangoapps/config_models/models.py index 5528f084cf..ab429c701d 100644 --- a/common/djangoapps/config_models/models.py +++ b/common/djangoapps/config_models/models.py @@ -6,6 +6,9 @@ from django.contrib.auth.models import User from django.core.cache import caches, InvalidCacheBackendError from django.utils.translation import ugettext_lazy as _ +from rest_framework.utils import model_meta + + try: cache = caches['configuration'] # pylint: disable=invalid-name except InvalidCacheBackendError: @@ -176,3 +179,58 @@ class ConfigurationModel(models.Model): values = list(cls.objects.values_list(*key_fields, flat=flat).order_by().distinct()) cache.set(cache_key, values, cls.cache_timeout) return values + + def fields_equal(self, instance, fields_to_ignore=("id", "change_date", "changed_by")): + """ + Compares this instance's fields to the supplied instance to test for equality. + This will ignore any fields in `fields_to_ignore`. + + Note that this method ignores many-to-many fields. + + Args: + instance: the model instance to compare + fields_to_ignore: List of fields that should not be compared for equality. By default + includes `id`, `change_date`, and `changed_by`. + + Returns: True if the checked fields are all equivalent, else False + """ + for field in self._meta.get_fields(): + if not field.many_to_many and field.name not in fields_to_ignore: + if getattr(instance, field.name) != getattr(self, field.name): + return False + + return True + + @classmethod + def equal_to_current(cls, json, fields_to_ignore=("id", "change_date", "changed_by")): + """ + Compares for equality this instance to a model instance constructed from the supplied JSON. + This will ignore any fields in `fields_to_ignore`. + + Note that this method cannot handle fields with many-to-many associations, as those can only + be set on a saved model instance (and saving the model instance will create a new entry). + All many-to-many field entries will be removed before the equality comparison is done. + + Args: + json: json representing an entry to compare + fields_to_ignore: List of fields that should not be compared for equality. By default + includes `id`, `change_date`, and `changed_by`. + + Returns: True if the checked fields are all equivalent, else False + """ + + # Remove many-to-many relationships from json. + # They require an instance to be already saved. + info = model_meta.get_field_info(cls) + for field_name, relation_info in info.relations.items(): + if relation_info.to_many and (field_name in json): + json.pop(field_name) + + new_instance = cls(**json) + key_field_args = tuple(getattr(new_instance, key) for key in cls.KEY_FIELDS) + current = cls.current(*key_field_args) + # If current.id is None, no entry actually existed and the "current" method created it. + if current.id is not None: + return current.fields_equal(new_instance, fields_to_ignore) + + return False diff --git a/common/djangoapps/config_models/tests/__init__.py b/common/djangoapps/config_models/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/common/djangoapps/config_models/tests/data/data.json b/common/djangoapps/config_models/tests/data/data.json new file mode 100644 index 0000000000..e6977c7d54 --- /dev/null +++ b/common/djangoapps/config_models/tests/data/data.json @@ -0,0 +1,14 @@ +{ + "model": "config_models.ExampleDeserializeConfig", + "data": [ + { + "name": "betty", + "enabled": true, + "int_field": 5 + }, + { + "name": "fred", + "enabled": false + } + ] +} diff --git a/common/djangoapps/config_models/tests/test_model_deserialization.py b/common/djangoapps/config_models/tests/test_model_deserialization.py new file mode 100644 index 0000000000..af1ff81e49 --- /dev/null +++ b/common/djangoapps/config_models/tests/test_model_deserialization.py @@ -0,0 +1,219 @@ +""" +Tests of the populate_model management command and its helper utils.deserialize_json method. +""" + +import textwrap +import os.path + +from django.utils import timezone +from django.utils.six import BytesIO + +from django.contrib.auth.models import User +from django.core.management.base import CommandError +from django.db import models + +from config_models.management.commands import populate_model +from config_models.models import ConfigurationModel +from config_models.utils import deserialize_json +from openedx.core.djangolib.testing.utils import CacheIsolationTestCase + + +class ExampleDeserializeConfig(ConfigurationModel): + """ + Test model for testing deserialization of ``ConfigurationModels`` with keyed configuration. + """ + KEY_FIELDS = ('name',) + + name = models.TextField() + int_field = models.IntegerField(default=10) + + def __unicode__(self): + return "ExampleDeserializeConfig(enabled={}, name={}, int_field={})".format( + self.enabled, self.name, self.int_field + ) + + +class DeserializeJSONTests(CacheIsolationTestCase): + """ + Tests of deserializing the JSON representation of ConfigurationModels. + """ + def setUp(self): + super(DeserializeJSONTests, self).setUp() + self.test_username = 'test_worker' + User.objects.create_user(username=self.test_username) + self.fixture_path = os.path.join(os.path.dirname(__file__), 'data', 'data.json') + + def test_deserialize_models(self): + """ + Tests the "happy path", where 2 instances of the test model should be created. + A valid username is supplied for the operation. + """ + start_date = timezone.now() + with open(self.fixture_path) as data: + entries_created = deserialize_json(data, self.test_username) + self.assertEquals(2, entries_created) + + self.assertEquals(2, ExampleDeserializeConfig.objects.count()) + + betty = ExampleDeserializeConfig.current('betty') + self.assertTrue(betty.enabled) + self.assertEquals(5, betty.int_field) + self.assertGreater(betty.change_date, start_date) + self.assertEquals(self.test_username, betty.changed_by.username) + + fred = ExampleDeserializeConfig.current('fred') + self.assertFalse(fred.enabled) + self.assertEquals(10, fred.int_field) + self.assertGreater(fred.change_date, start_date) + self.assertEquals(self.test_username, fred.changed_by.username) + + def test_existing_entries_not_removed(self): + """ + Any existing configuration model entries are retained + (though they may be come history)-- deserialize_json is purely additive. + """ + ExampleDeserializeConfig(name="fred", enabled=True).save() + ExampleDeserializeConfig(name="barney", int_field=200).save() + + with open(self.fixture_path) as data: + entries_created = deserialize_json(data, self.test_username) + self.assertEquals(2, entries_created) + + self.assertEquals(4, ExampleDeserializeConfig.objects.count()) + self.assertEquals(3, len(ExampleDeserializeConfig.objects.current_set())) + + self.assertEquals(5, ExampleDeserializeConfig.current('betty').int_field) + self.assertEquals(200, ExampleDeserializeConfig.current('barney').int_field) + + # The JSON file changes "enabled" to False for Fred. + fred = ExampleDeserializeConfig.current('fred') + self.assertFalse(fred.enabled) + + def test_duplicate_entries_not_made(self): + """ + If there is no change in an entry (besides changed_by and change_date), + a new entry is not made. + """ + with open(self.fixture_path) as data: + entries_created = deserialize_json(data, self.test_username) + self.assertEquals(2, entries_created) + + with open(self.fixture_path) as data: + entries_created = deserialize_json(data, self.test_username) + self.assertEquals(0, entries_created) + + # Importing twice will still only result in 2 records (second import a no-op). + self.assertEquals(2, ExampleDeserializeConfig.objects.count()) + + # Change Betty. + betty = ExampleDeserializeConfig.current('betty') + betty.int_field = -8 + betty.save() + + self.assertEquals(3, ExampleDeserializeConfig.objects.count()) + self.assertEquals(-8, ExampleDeserializeConfig.current('betty').int_field) + + # Now importing will add a new entry for Betty. + with open(self.fixture_path) as data: + entries_created = deserialize_json(data, self.test_username) + self.assertEquals(1, entries_created) + + self.assertEquals(4, ExampleDeserializeConfig.objects.count()) + self.assertEquals(5, ExampleDeserializeConfig.current('betty').int_field) + + def test_bad_username(self): + """ + Tests the error handling when the specified user does not exist. + """ + test_json = textwrap.dedent(""" + { + "model": "config_models.ExampleDeserializeConfig", + "data": [{"name": "dino"}] + } + """) + with self.assertRaisesRegexp(Exception, "User matching query does not exist"): + deserialize_json(BytesIO(test_json), "unknown_username") + + def test_invalid_json(self): + """ + Tests the error handling when there is invalid JSON. + """ + test_json = textwrap.dedent(""" + { + "model": "config_models.ExampleDeserializeConfig", + "data": [{"name": "dino" + """) + with self.assertRaisesRegexp(Exception, "JSON parse error"): + deserialize_json(BytesIO(test_json), self.test_username) + + def test_invalid_model(self): + """ + Tests the error handling when the configuration model specified does not exist. + """ + test_json = textwrap.dedent(""" + { + "model": "xxx.yyy", + "data":[{"name": "dino"}] + } + """) + with self.assertRaisesRegexp(Exception, "No installed app"): + deserialize_json(BytesIO(test_json), self.test_username) + + +class PopulateModelTestCase(CacheIsolationTestCase): + """ + Tests of populate model management command. + """ + def setUp(self): + super(PopulateModelTestCase, self).setUp() + self.file_path = os.path.join(os.path.dirname(__file__), 'data', 'data.json') + self.test_username = 'test_management_worker' + User.objects.create_user(username=self.test_username) + + def test_run_command(self): + """ + Tests the "happy path", where 2 instances of the test model should be created. + A valid username is supplied for the operation. + """ + _run_command(file=self.file_path, username=self.test_username) + self.assertEquals(2, ExampleDeserializeConfig.objects.count()) + + betty = ExampleDeserializeConfig.current('betty') + self.assertEquals(self.test_username, betty.changed_by.username) + + fred = ExampleDeserializeConfig.current('fred') + self.assertEquals(self.test_username, fred.changed_by.username) + + def test_no_user_specified(self): + """ + Tests that a username must be specified. + """ + with self.assertRaisesRegexp(CommandError, "A valid username must be specified"): + _run_command(file=self.file_path) + + def test_bad_user_specified(self): + """ + Tests that a username must be specified. + """ + with self.assertRaisesRegexp(Exception, "User matching query does not exist"): + _run_command(file=self.file_path, username="does_not_exist") + + def test_no_file_specified(self): + """ + Tests the error handling when no JSON file is supplied. + """ + with self.assertRaisesRegexp(CommandError, "A file containing JSON must be specified"): + _run_command(username=self.test_username) + + def test_bad_file_specified(self): + """ + Tests the error handling when the path to the JSON file is incorrect. + """ + with self.assertRaisesRegexp(CommandError, "File does/not/exist.json does not exist"): + _run_command(file="does/not/exist.json", username=self.test_username) + + +def _run_command(*args, **kwargs): + """Run the management command to deserializer JSON ConfigurationModel data. """ + command = populate_model.Command() + return command.handle(*args, **kwargs) diff --git a/common/djangoapps/config_models/tests.py b/common/djangoapps/config_models/tests/tests.py similarity index 73% rename from common/djangoapps/config_models/tests.py rename to common/djangoapps/config_models/tests/tests.py index 15f954e336..538058c109 100644 --- a/common/djangoapps/config_models/tests.py +++ b/common/djangoapps/config_models/tests/tests.py @@ -25,6 +25,24 @@ class ExampleConfig(ConfigurationModel): string_field = models.TextField() int_field = models.IntegerField(default=10) + def __unicode__(self): + return "ExampleConfig(enabled={}, string_field={}, int_field={})".format( + self.enabled, self.string_field, self.int_field + ) + + +class ManyToManyExampleConfig(ConfigurationModel): + """ + Test model configuration with a many-to-many field. + """ + cache_timeout = 300 + + string_field = models.TextField() + many_user_field = models.ManyToManyField(User, related_name='topic_many_user_field') + + def __unicode__(self): + return "ManyToManyExampleConfig(enabled={}, string_field={})".format(self.enabled, self.string_field) + @patch('config_models.models.cache') class ConfigurationModelTests(TestCase): @@ -40,7 +58,7 @@ class ConfigurationModelTests(TestCase): ExampleConfig(changed_by=self.user).save() mock_cache.delete.assert_called_with(ExampleConfig.cache_key_name()) - def test_cache_key_name(self, _mock_cache): + def test_cache_key_name(self, __): self.assertEquals(ExampleConfig.cache_key_name(), 'configuration/ExampleConfig/current') def test_no_config_empty_cache(self, mock_cache): @@ -103,6 +121,64 @@ class ConfigurationModelTests(TestCase): self.assertEquals(2, ExampleConfig.objects.all().count()) + def test_equality(self, mock_cache): + mock_cache.get.return_value = None + + config = ExampleConfig(changed_by=self.user, string_field='first') + config.save() + + self.assertTrue(ExampleConfig.equal_to_current({"string_field": "first"})) + self.assertTrue(ExampleConfig.equal_to_current({"string_field": "first", "enabled": False})) + self.assertTrue(ExampleConfig.equal_to_current({"string_field": "first", "int_field": 10})) + + self.assertFalse(ExampleConfig.equal_to_current({"string_field": "first", "enabled": True})) + self.assertFalse(ExampleConfig.equal_to_current({"string_field": "first", "int_field": 20})) + self.assertFalse(ExampleConfig.equal_to_current({"string_field": "second"})) + + self.assertFalse(ExampleConfig.equal_to_current({})) + + def test_equality_custom_fields_to_ignore(self, mock_cache): + mock_cache.get.return_value = None + + config = ExampleConfig(changed_by=self.user, string_field='first') + config.save() + + # id, change_date, and changed_by will all be different for a newly created entry + self.assertTrue(ExampleConfig.equal_to_current({"string_field": "first"})) + self.assertFalse( + ExampleConfig.equal_to_current({"string_field": "first"}, fields_to_ignore=("change_date", "changed_by")) + ) + self.assertFalse( + ExampleConfig.equal_to_current({"string_field": "first"}, fields_to_ignore=("id", "changed_by")) + ) + self.assertFalse( + ExampleConfig.equal_to_current({"string_field": "first"}, fields_to_ignore=("change_date", "id")) + ) + + # Test the ability to ignore a different field ("int_field"). + self.assertFalse(ExampleConfig.equal_to_current({"string_field": "first", "int_field": 20})) + self.assertTrue( + ExampleConfig.equal_to_current( + {"string_field": "first", "int_field": 20}, + fields_to_ignore=("id", "change_date", "changed_by", "int_field") + ) + ) + + def test_equality_ignores_many_to_many(self, mock_cache): + mock_cache.get.return_value = None + config = ManyToManyExampleConfig(changed_by=self.user, string_field='first') + config.save() + + second_user = User(username="second_user") + second_user.save() + config.many_user_field.add(second_user) # pylint: disable=no-member + config.save() + + # The many-to-many field is ignored in comparison. + self.assertTrue( + ManyToManyExampleConfig.equal_to_current({"string_field": "first", "many_user_field": "removed"}) + ) + class ExampleKeyedConfig(ConfigurationModel): """ @@ -120,6 +196,11 @@ class ExampleKeyedConfig(ConfigurationModel): string_field = models.TextField() int_field = models.IntegerField(default=10) + def __unicode__(self): + return "ExampleKeyedConfig(enabled={}, left={}, right={}, string_field={}, int_field={})".format( + self.enabled, self.left, self.right, self.string_field, self.int_field + ) + @ddt.ddt @patch('config_models.models.cache') @@ -294,6 +375,45 @@ class KeyedConfigurationModelTests(TestCase): mock_cache.get.return_value = fake_result self.assertEquals(ExampleKeyedConfig.key_values(), fake_result) + def test_equality(self, mock_cache): + mock_cache.get.return_value = None + + config1 = ExampleKeyedConfig(left='left_a', right='right_a', int_field=1, changed_by=self.user) + config1.save() + + config2 = ExampleKeyedConfig(left='left_b', right='right_b', int_field=2, changed_by=self.user, enabled=True) + config2.save() + + config3 = ExampleKeyedConfig(left='left_c', changed_by=self.user) + config3.save() + + self.assertTrue( + ExampleKeyedConfig.equal_to_current({"left": "left_a", "right": "right_a", "int_field": 1}) + ) + self.assertTrue( + ExampleKeyedConfig.equal_to_current({"left": "left_b", "right": "right_b", "int_field": 2, "enabled": True}) + ) + self.assertTrue( + ExampleKeyedConfig.equal_to_current({"left": "left_c"}) + ) + + self.assertFalse( + ExampleKeyedConfig.equal_to_current( + {"left": "left_a", "right": "right_a", "int_field": 1, "string_field": "foo"} + ) + ) + self.assertFalse( + ExampleKeyedConfig.equal_to_current({"left": "left_a", "int_field": 1}) + ) + self.assertFalse( + ExampleKeyedConfig.equal_to_current({"left": "left_b", "right": "right_b", "int_field": 2}) + ) + self.assertFalse( + ExampleKeyedConfig.equal_to_current({"left": "left_c", "int_field": 11}) + ) + + self.assertFalse(ExampleKeyedConfig.equal_to_current({})) + @ddt.ddt class ConfigurationModelAPITests(TestCase): diff --git a/common/djangoapps/config_models/utils.py b/common/djangoapps/config_models/utils.py new file mode 100644 index 0000000000..10a293af4d --- /dev/null +++ b/common/djangoapps/config_models/utils.py @@ -0,0 +1,69 @@ +""" +Utilities for working with ConfigurationModels. +""" +from django.apps import apps +from rest_framework.parsers import JSONParser +from rest_framework.serializers import ModelSerializer +from django.contrib.auth.models import User + + +def get_serializer_class(configuration_model): + """ Returns a ConfigurationModel serializer class for the supplied configuration_model. """ + class AutoConfigModelSerializer(ModelSerializer): + """Serializer class for configuration models.""" + + class Meta(object): + """Meta information for AutoConfigModelSerializer.""" + model = configuration_model + + def create(self, validated_data): + if "changed_by_username" in self.context: + validated_data['changed_by'] = User.objects.get(username=self.context["changed_by_username"]) + return super(AutoConfigModelSerializer, self).create(validated_data) + + return AutoConfigModelSerializer + + +def deserialize_json(stream, username): + """ + Given a stream containing JSON, deserializers the JSON into ConfigurationModel instances. + + The stream is expected to be in the following format: + { "model": "config_models.ExampleConfigurationModel", + "data": + [ + { "enabled": True, + "color": "black" + ... + }, + { "enabled": False, + "color": "yellow" + ... + }, + ... + ] + } + + If the provided stream does not contain valid JSON for the ConfigurationModel specified, + an Exception will be raised. + + Arguments: + stream: The stream of JSON, as described above. + username: The username of the user making the change. This must match an existing user. + + Returns: the number of created entries + """ + parsed_json = JSONParser().parse(stream) + serializer_class = get_serializer_class(apps.get_model(parsed_json["model"])) + list_serializer = serializer_class(data=parsed_json["data"], context={"changed_by_username": username}, many=True) + if list_serializer.is_valid(): + model_class = serializer_class.Meta.model + for data in reversed(list_serializer.validated_data): + if model_class.equal_to_current(data): + list_serializer.validated_data.remove(data) + + entries_created = len(list_serializer.validated_data) + list_serializer.save() + return entries_created + else: + raise Exception(list_serializer.error_messages) diff --git a/common/djangoapps/config_models/views.py b/common/djangoapps/config_models/views.py index 3bd693ec59..c9d584f9cb 100644 --- a/common/djangoapps/config_models/views.py +++ b/common/djangoapps/config_models/views.py @@ -4,9 +4,10 @@ API view to allow manipulation of configuration models. from rest_framework.generics import CreateAPIView, RetrieveAPIView from rest_framework.permissions import DjangoModelPermissions from rest_framework.authentication import SessionAuthentication -from rest_framework.serializers import ModelSerializer from django.db import transaction +from config_models.utils import get_serializer_class + class ReadableOnlyByAuthors(DjangoModelPermissions): """Only allow access by users with `add` permissions on the model.""" @@ -58,13 +59,7 @@ class ConfigurationModelCurrentAPIView(AtomicMixin, CreateAPIView, RetrieveAPIVi def get_serializer_class(self): if self.serializer_class is None: - class AutoConfigModelSerializer(ModelSerializer): - """Serializer class for configuration models.""" - class Meta(object): - """Meta information for AutoConfigModelSerializer.""" - model = self.model - - self.serializer_class = AutoConfigModelSerializer + self.serializer_class = get_serializer_class(self.model) return self.serializer_class