Finishing async course structure work
- Added tests - Updated model field specification - Fixed issue of multiple event emission - Updated admin page - Added management command to manually generate course structures
This commit is contained in:
Clinton Blackburn
@@ -1,14 +1,12 @@
|
||||
import json
|
||||
|
||||
from ratelimitbackend import admin
|
||||
|
||||
from .models import CourseStructure
|
||||
|
||||
|
||||
class CourseStructureAdmin(admin.ModelAdmin):
|
||||
search_fields = ('course_id', 'version')
|
||||
list_display = (
|
||||
'id', 'course_id', 'version', 'created'
|
||||
)
|
||||
list_display_links = ('id', 'course_id')
|
||||
search_fields = ('course_id',)
|
||||
list_display = ('course_id', 'modified')
|
||||
ordering = ('course_id', '-modified')
|
||||
|
||||
|
||||
admin.site.register(CourseStructure, CourseStructureAdmin)
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import logging
|
||||
from optparse import make_option
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from opaque_keys.edx.keys import CourseKey
|
||||
from xmodule.modulestore.django import modulestore
|
||||
|
||||
from openedx.core.djangoapps.content.course_structures.models import update_course_structure
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
args = '<course_id course_id ...>'
|
||||
help = 'Generates and stores course structure for one or more courses.'
|
||||
|
||||
option_list = BaseCommand.option_list + (
|
||||
make_option('--all',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Generate structures for all courses.'),
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
|
||||
if options['all']:
|
||||
course_keys = [course.id for course in modulestore().get_courses()]
|
||||
else:
|
||||
course_keys = [CourseKey.from_string(arg) for arg in args]
|
||||
|
||||
if not course_keys:
|
||||
logger.fatal('No courses specified.')
|
||||
return
|
||||
|
||||
logger.info('Generating course structures for %d courses.', len(course_keys))
|
||||
logging.debug('Generating course structure(s) for the following courses: %s', course_keys)
|
||||
|
||||
for course_key in course_keys:
|
||||
try:
|
||||
update_course_structure(course_key)
|
||||
except Exception as e:
|
||||
logger.error('An error occurred while generating course structure for %s: %s', unicode(course_key), e)
|
||||
|
||||
logger.info('Finished generating course structures.')
|
||||
@@ -13,9 +13,8 @@ class Migration(SchemaMigration):
|
||||
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
|
||||
('created', self.gf('model_utils.fields.AutoCreatedField')(default=datetime.datetime.now)),
|
||||
('modified', self.gf('model_utils.fields.AutoLastModifiedField')(default=datetime.datetime.now)),
|
||||
('course_id', self.gf('xmodule_django.models.CourseKeyField')(max_length=255, db_index=True)),
|
||||
('version', self.gf('django.db.models.fields.CharField')(max_length=255)),
|
||||
('structure_json', self.gf('django.db.models.fields.TextField')()),
|
||||
('course_id', self.gf('xmodule_django.models.CourseKeyField')(unique=True, max_length=255, db_index=True)),
|
||||
('structure_json', self.gf('django.db.models.fields.TextField')(null=True, blank=True)),
|
||||
))
|
||||
db.send_create_signal('course_structures', ['CourseStructure'])
|
||||
|
||||
@@ -28,12 +27,11 @@ class Migration(SchemaMigration):
|
||||
models = {
|
||||
'course_structures.coursestructure': {
|
||||
'Meta': {'object_name': 'CourseStructure'},
|
||||
'course_id': ('xmodule_django.models.CourseKeyField', [], {'max_length': '255', 'db_index': 'True'}),
|
||||
'course_id': ('xmodule_django.models.CourseKeyField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}),
|
||||
'created': ('model_utils.fields.AutoCreatedField', [], {'default': 'datetime.datetime.now'}),
|
||||
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'modified': ('model_utils.fields.AutoLastModifiedField', [], {'default': 'datetime.datetime.now'}),
|
||||
'structure_json': ('django.db.models.fields.TextField', [], {}),
|
||||
'version': ('django.db.models.fields.CharField', [], {'max_length': '255'})
|
||||
'structure_json': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,46 +1,53 @@
|
||||
import json
|
||||
import logging
|
||||
|
||||
from django.db import models
|
||||
from django.dispatch import receiver
|
||||
from celery.task import task
|
||||
from django.dispatch import receiver
|
||||
from model_utils.models import TimeStampedModel
|
||||
|
||||
from opaque_keys.edx.keys import CourseKey
|
||||
|
||||
from opaque_keys.edx.locator import CourseLocator
|
||||
from xmodule.modulestore.django import modulestore, SignalHandler
|
||||
|
||||
from util.models import CompressedTextField
|
||||
from xmodule_django.models import CourseKeyField
|
||||
|
||||
class CourseStructure(TimeStampedModel):
|
||||
|
||||
course_id = CourseKeyField(max_length=255, db_index=True)
|
||||
version = models.CharField(max_length=255, blank=True, default="")
|
||||
logger = logging.getLogger(__name__) # pylint: disable=invalid-name
|
||||
|
||||
|
||||
class CourseStructure(TimeStampedModel):
|
||||
course_id = CourseKeyField(max_length=255, db_index=True, unique=True, verbose_name='Course ID')
|
||||
|
||||
# Right now the only thing we do with the structure doc is store it and
|
||||
# send it on request. If we need to store a more complex data model later,
|
||||
# we can do so and build a migration. The only problem with a normalized
|
||||
# data model for this is that it will likely involve hundreds of rows, and
|
||||
# we'd have to be careful about caching.
|
||||
structure_json = models.TextField()
|
||||
structure_json = CompressedTextField(verbose_name='Structure JSON', blank=True, null=True)
|
||||
|
||||
# Index together:
|
||||
# (course_id, version)
|
||||
# (course_id, created)
|
||||
@property
|
||||
def structure(self):
|
||||
if self.structure_json:
|
||||
return json.loads(self.structure_json)
|
||||
return None
|
||||
|
||||
|
||||
def course_structure(course_key):
|
||||
def generate_course_structure(course_key):
|
||||
"""
|
||||
Generates a course structure dictionary for the specified course.
|
||||
"""
|
||||
course = modulestore().get_course(course_key, depth=None)
|
||||
blocks_stack = [course]
|
||||
blocks_dict = {}
|
||||
while blocks_stack:
|
||||
curr_block = blocks_stack.pop()
|
||||
children = curr_block.get_children() if curr_block.has_children else []
|
||||
children = curr_block.get_children() if curr_block.has_children else []
|
||||
blocks_dict[unicode(curr_block.scope_ids.usage_id)] = {
|
||||
"usage_key": unicode(curr_block.scope_ids.usage_id),
|
||||
"block_type": curr_block.category,
|
||||
"display_name": curr_block.display_name,
|
||||
"graded": curr_block.graded,
|
||||
"format": curr_block.format,
|
||||
"children": [unicode(ch.scope_ids.usage_id) for ch in children]
|
||||
"children": [unicode(child.scope_ids.usage_id) for child in children]
|
||||
}
|
||||
blocks_stack.extend(children)
|
||||
return {
|
||||
@@ -48,15 +55,38 @@ def course_structure(course_key):
|
||||
"blocks": blocks_dict
|
||||
}
|
||||
|
||||
|
||||
@receiver(SignalHandler.course_published)
|
||||
def listen_for_course_publish(sender, course_key, **kwargs):
|
||||
update_course_structure(course_key)
|
||||
# Note: The countdown=0 kwarg is set to to ensure the method below does not attempt to access the course
|
||||
# before the signal emitter has finished all operations. This is also necessary to ensure all tests pass.
|
||||
update_course_structure.delay(course_key, countdown=0)
|
||||
|
||||
|
||||
@task()
|
||||
def update_course_structure(course_key):
|
||||
structure = course_structure(course_key)
|
||||
CourseStructure.objects.create(
|
||||
course_id=unicode(course_key),
|
||||
structure_json=json.dumps(structure),
|
||||
version="",
|
||||
"""
|
||||
Regenerates and updates the course structure (in the database) for the specified course.
|
||||
"""
|
||||
if not isinstance(course_key, CourseLocator):
|
||||
logger.error('update_course_structure requires a CourseLocator. Given %s.', type(course_key))
|
||||
return
|
||||
|
||||
try:
|
||||
structure = generate_course_structure(course_key)
|
||||
except Exception as e:
|
||||
logger.error('An error occurred while generating course structure: %s', e)
|
||||
raise
|
||||
|
||||
structure_json = json.dumps(structure)
|
||||
|
||||
cs, created = CourseStructure.objects.get_or_create(
|
||||
course_id=course_key,
|
||||
defaults={'structure_json': structure_json}
|
||||
)
|
||||
|
||||
if not created:
|
||||
cs.structure_json = structure_json
|
||||
cs.save()
|
||||
|
||||
return cs
|
||||
|
||||
79
openedx/core/djangoapps/content/course_structures/tests.py
Normal file
79
openedx/core/djangoapps/content/course_structures/tests.py
Normal file
@@ -0,0 +1,79 @@
|
||||
import json
|
||||
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
|
||||
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
|
||||
|
||||
from openedx.core.djangoapps.content.course_structures.models import generate_course_structure, CourseStructure
|
||||
|
||||
|
||||
class CourseStructureTests(ModuleStoreTestCase):
|
||||
def setUp(self, **kwargs):
|
||||
super(CourseStructureTests, self).setUp()
|
||||
self.course = CourseFactory.create()
|
||||
self.section = ItemFactory.create(parent=self.course, category='chapter', display_name='Test Section')
|
||||
CourseStructure.objects.all().delete()
|
||||
|
||||
def test_generate_course_structure(self):
|
||||
blocks = {}
|
||||
|
||||
def add_block(block):
|
||||
children = block.get_children() if block.has_children else []
|
||||
|
||||
blocks[unicode(block.location)] = {
|
||||
"usage_key": unicode(block.location),
|
||||
"block_type": block.category,
|
||||
"display_name": block.display_name,
|
||||
"graded": block.graded,
|
||||
"format": block.format,
|
||||
"children": [unicode(child.location) for child in children]
|
||||
}
|
||||
|
||||
for child in children:
|
||||
add_block(child)
|
||||
|
||||
add_block(self.course)
|
||||
|
||||
expected = {
|
||||
'root': unicode(self.course.location),
|
||||
'blocks': blocks
|
||||
}
|
||||
|
||||
self.maxDiff = None
|
||||
actual = generate_course_structure(self.course.id)
|
||||
self.assertDictEqual(actual, expected)
|
||||
|
||||
def test_structure_json(self):
|
||||
"""
|
||||
Although stored as compressed data, CourseStructure.structure_json should always return the uncompressed string.
|
||||
"""
|
||||
course_id = 'a/b/c'
|
||||
structure = {
|
||||
'root': course_id,
|
||||
'blocks': {
|
||||
course_id: {
|
||||
'id': course_id
|
||||
}
|
||||
}
|
||||
}
|
||||
structure_json = json.dumps(structure)
|
||||
cs = CourseStructure.objects.create(course_id=self.course.id, structure_json=structure_json)
|
||||
self.assertEqual(cs.structure_json, structure_json)
|
||||
|
||||
# Reload the data to ensure the init signal is fired to decompress the data.
|
||||
cs = CourseStructure.objects.get(course_id=self.course.id)
|
||||
self.assertEqual(cs.structure_json, structure_json)
|
||||
|
||||
def test_structure(self):
|
||||
"""
|
||||
CourseStructure.structure should return the uncompressed, JSON-parsed course structure.
|
||||
"""
|
||||
structure = {
|
||||
'root': 'a/b/c',
|
||||
'blocks': {
|
||||
'a/b/c': {
|
||||
'id': 'a/b/c'
|
||||
}
|
||||
}
|
||||
}
|
||||
structure_json = json.dumps(structure)
|
||||
cs = CourseStructure.objects.create(course_id=self.course.id, structure_json=structure_json)
|
||||
self.assertDictEqual(cs.structure, structure)
|
||||
Reference in New Issue
Block a user