From 085cc17178658af9df18bc72ebad6ec9b8810a38 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Thu, 16 Feb 2012 19:52:52 -0500 Subject: [PATCH 1/4] Caching problem maximum scores in profile (~10x performance boost in that section of the code) --HG-- branch : profile-cache --- .../migrations/0003_done_grade_cache.py | 116 ++++++++++++++++++ courseware/models.py | 9 +- courseware/views.py | 48 +++++--- 3 files changed, 156 insertions(+), 17 deletions(-) create mode 100644 courseware/migrations/0003_done_grade_cache.py diff --git a/courseware/migrations/0003_done_grade_cache.py b/courseware/migrations/0003_done_grade_cache.py new file mode 100644 index 0000000000..f2fedcf1a8 --- /dev/null +++ b/courseware/migrations/0003_done_grade_cache.py @@ -0,0 +1,116 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Removing unique constraint on 'StudentModule', fields ['module_id', 'module_type', 'student'] + db.delete_unique('courseware_studentmodule', ['module_id', 'module_type', 'student_id']) + + # Adding field 'StudentModule.max_grade' + db.add_column('courseware_studentmodule', 'max_grade', self.gf('django.db.models.fields.FloatField')(null=True, blank=True), keep_default=False) + + # Adding field 'StudentModule.done' + db.add_column('courseware_studentmodule', 'done', self.gf('django.db.models.fields.CharField')(default='na', max_length=8, db_index=True), keep_default=False) + + # Adding unique constraint on 'StudentModule', fields ['module_id', 'student'] + db.create_unique('courseware_studentmodule', ['module_id', 'student_id']) + + + def backwards(self, orm): + + # Removing unique constraint on 'StudentModule', fields ['module_id', 'student'] + db.delete_unique('courseware_studentmodule', ['module_id', 'student_id']) + + # Deleting field 'StudentModule.max_grade' + db.delete_column('courseware_studentmodule', 'max_grade') + + # Deleting field 'StudentModule.done' + db.delete_column('courseware_studentmodule', 'done') + + # Adding unique constraint on 'StudentModule', fields ['module_id', 'module_type', 'student'] + db.create_unique('courseware_studentmodule', ['module_id', 'module_type', 'student_id']) + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'about': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'avatar_type': ('django.db.models.fields.CharField', [], {'default': "'n'", 'max_length': '1'}), + 'bronze': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}), + 'consecutive_days_visit_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2', 'blank': 'True'}), + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'date_of_birth': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'display_tag_filter_strategy': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'email_isvalid': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'email_key': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}), + 'email_tag_filter_strategy': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'gold': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}), + 'gravatar': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ignored_tags': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'interesting_tags': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'last_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'location': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}), + 'new_response_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'questions_per_page': ('django.db.models.fields.SmallIntegerField', [], {'default': '10'}), + 'real_name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}), + 'reputation': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}), + 'seen_response_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'show_country': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'silver': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'w'", 'max_length': '2'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}), + 'website': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'courseware.studentmodule': { + 'Meta': {'unique_together': "(('student', 'module_id'),)", 'object_name': 'StudentModule'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}), + 'done': ('django.db.models.fields.CharField', [], {'default': "'na'", 'max_length': '8', 'db_index': 'True'}), + 'grade': ('django.db.models.fields.FloatField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'max_grade': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}), + 'module_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'module_type': ('django.db.models.fields.CharField', [], {'default': "'problem'", 'max_length': '32', 'db_index': 'True'}), + 'state': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'student': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + } + } + + complete_apps = ['courseware'] diff --git a/courseware/models.py b/courseware/models.py index f713da3195..a41e58eb81 100644 --- a/courseware/models.py +++ b/courseware/models.py @@ -31,8 +31,13 @@ class StudentModule(models.Model): ## Grade, and are we done? grade = models.FloatField(null=True, blank=True, db_index=True) - #max_grade = models.FloatField(null=True, blank=True) - + max_grade = models.FloatField(null=True, blank=True) + DONE_TYPES = (('na','NOT_APPLICABLE'), + ('f','FINISHED'), + ('i','INCOMPLETE'), + ) + done = models.CharField(max_length=8, choices=DONE_TYPES, default='na', db_index=True) + # DONE_TYPES = (('done','DONE'), # Finished # ('incomplete','NOTDONE'), # Not finished # ('na','NA')) # Not applicable (e.g. vertical) diff --git a/courseware/views.py b/courseware/views.py index fbdd7fceab..b3253af592 100644 --- a/courseware/views.py +++ b/courseware/views.py @@ -33,6 +33,22 @@ etree.set_default_parser(etree.XMLParser(dtd_validation=False, load_dtd=False, template_imports={'urllib':urllib} +def get_grade(request, problem, cache): + id = problem.get('id') + correct = 0 + if id in cache: + response = cache[id] + if response.grade!=None: + correct=response.grade + if response.max_grade != None: + total = response.max_grade + else: + total=courseware.modules.capa_module.Module(etree.tostring(problem), "id").max_score() # TODO: Add state. Not useful now, but maybe someday problems will have randomized max scores? + response.max_grade = total + response.save() + + return (correct, total) + @cache_control(no_cache=True, no_store=True, must_revalidate=True) def profile(request): ''' User profile. Show username, location, etc, as well as grades . @@ -59,28 +75,30 @@ def profile(request): course=course, chname=chname): problems=dom.xpath('//course[@name=$course]/chapter[@name=$chname]/section[@name=$section]//problem', course=course, chname=chname, section=s.get('name')) - + graded = True if s.get('graded') == "true" else False scores=[] if len(problems)>0: for p in problems: - id = p.get('id') - correct = 0 - if id in response_by_id: - response = response_by_id[id] - if response.grade!=None: - correct=response.grade - - total=courseware.modules.capa_module.Module(etree.tostring(p), "id").max_score() # TODO: Add state. Not useful now, but maybe someday problems will have randomized max scores? + (correct,total) = get_grade(request, p, response_by_id) + # id = p.get('id') + # correct = 0 + # if id in response_by_id: + # response = response_by_id[id] + # if response.grade!=None: + # correct=response.grade + + # total=courseware.modules.capa_module.Module(etree.tostring(p), "id").max_score() # TODO: Add state. Not useful now, but maybe someday problems will have randomized max scores? + # print correct, total scores.append((int(correct),total, graded )) - - + + section_total = (sum([score[0] for score in scores]), sum([score[1] for score in scores])) - + graded_total = (sum([score[0] for score in scores if score[2]]), sum([score[1] for score in scores if score[2]])) - + #Add the graded total to total_scores format = s.get('format') if s.get('format') else "" subtitle = s.get('subtitle') if s.get('subtitle') else format @@ -88,7 +106,7 @@ def profile(request): format_scores = total_scores[ format ] if format in total_scores else [] format_scores.append( graded_total ) total_scores[ format ] = format_scores - + score={'section':s.get("name"), 'scores':scores, 'section_total' : section_total, @@ -98,7 +116,7 @@ def profile(request): 'graded' : graded, } sections.append(score) - + chapters.append({'course':course, 'chapter' : c.get("name"), 'sections' : sections,}) From b2f3a75d861479b6bce4022ef228dbdfe2b7a174 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 19 Feb 2012 12:18:19 -0500 Subject: [PATCH 2/4] Working on case where id not in cache. Committing to merge with mainline --HG-- branch : profile-cache --- courseware/views.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/courseware/views.py b/courseware/views.py index b3253af592..4369816444 100644 --- a/courseware/views.py +++ b/courseware/views.py @@ -34,18 +34,33 @@ etree.set_default_parser(etree.XMLParser(dtd_validation=False, load_dtd=False, template_imports={'urllib':urllib} def get_grade(request, problem, cache): + ## HACK: assumes max score is fixed per problem id = problem.get('id') correct = 0 + print ",", if id in cache: + print "In cache" response = cache[id] if response.grade!=None: correct=response.grade - if response.max_grade != None: + if id in cache and response.max_grade != None: total = response.max_grade - else: - total=courseware.modules.capa_module.Module(etree.tostring(problem), "id").max_score() # TODO: Add state. Not useful now, but maybe someday problems will have randomized max scores? - response.max_grade = total - response.save() + else: #if id in cache and response.max_grade == None: + total=courseware.modules.capa_module.Module(etree.tostring(problem), "id").max_score() + # response.max_grade = total + # response.save() + # else: # if id not in cache + # total=courseware.modules.capa_module.Module(etree.tostring(problem), "id").max_score() + # module = StudentModule(module_type = 'problem', + # module_id = id, + # student = request.user, + # state = None, + # grade = 0, + # max_grade = total, + # done = 'i') + # total=courseware.modules.capa_module.Module(etree.tostring(problem), "id").max_score() + # module.save() + # cache[id] = module return (correct, total) From 7bea9382465a728bc8a960e8824873103fbf9f8f Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 19 Feb 2012 12:21:57 -0500 Subject: [PATCH 3/4] Handling case of response not in cache explicitly --HG-- branch : profile-cache --- courseware/views.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/courseware/views.py b/courseware/views.py index b4dc0683dd..c313cc43e3 100644 --- a/courseware/views.py +++ b/courseware/views.py @@ -38,30 +38,28 @@ def get_grade(request, problem, cache): ## HACK: assumes max score is fixed per problem id = problem.get('id') correct = 0 - print ",", if id in cache: - print "In cache" response = cache[id] if response.grade!=None: correct=response.grade if id in cache and response.max_grade != None: total = response.max_grade - else: #if id in cache and response.max_grade == None: + elif id in cache and response.max_grade == None: total=courseware.modules.capa_module.Module(etree.tostring(problem), "id").max_score() - # response.max_grade = total - # response.save() - # else: # if id not in cache - # total=courseware.modules.capa_module.Module(etree.tostring(problem), "id").max_score() - # module = StudentModule(module_type = 'problem', - # module_id = id, - # student = request.user, - # state = None, - # grade = 0, - # max_grade = total, - # done = 'i') - # total=courseware.modules.capa_module.Module(etree.tostring(problem), "id").max_score() - # module.save() - # cache[id] = module + response.max_grade = total + response.save() + else: # if id not in cache + total=courseware.modules.capa_module.Module(etree.tostring(problem), "id").max_score() + module = StudentModule(module_type = 'problem', + module_id = id, + student = request.user, + state = None, + grade = 0, + max_grade = total, + done = 'i') + total=courseware.modules.capa_module.Module(etree.tostring(problem), "id").max_score() + module.save() + cache[id] = module return (correct, total) From ab0f07ebd179426910f9f851fb5af4b5e5ff5c5d Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 19 Feb 2012 12:30:08 -0500 Subject: [PATCH 4/4] Code cleaned up to be slighlty more explicit --HG-- branch : profile-cache --- courseware/views.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/courseware/views.py b/courseware/views.py index c313cc43e3..2556cb44bb 100644 --- a/courseware/views.py +++ b/courseware/views.py @@ -38,29 +38,32 @@ def get_grade(request, problem, cache): ## HACK: assumes max score is fixed per problem id = problem.get('id') correct = 0 - if id in cache: - response = cache[id] - if response.grade!=None: - correct=response.grade - if id in cache and response.max_grade != None: - total = response.max_grade - elif id in cache and response.max_grade == None: - total=courseware.modules.capa_module.Module(etree.tostring(problem), "id").max_score() - response.max_grade = total - response.save() - else: # if id not in cache - total=courseware.modules.capa_module.Module(etree.tostring(problem), "id").max_score() - module = StudentModule(module_type = 'problem', + + # If the ID is not in the cache, add the item + if id not in cache: + module = StudentModule(module_type = 'problem', # TODO: Move into StudentModule.__init__? module_id = id, student = request.user, state = None, grade = 0, - max_grade = total, + max_grade = None, done = 'i') - total=courseware.modules.capa_module.Module(etree.tostring(problem), "id").max_score() - module.save() cache[id] = module + # Grab the # correct from cache + if id in cache: + response = cache[id] + if response.grade!=None: + correct=response.grade + + # Grab max grade from cache, or if it doesn't exist, compute and save to DB + if id in cache and response.max_grade != None: + total = response.max_grade + else: + total=courseware.modules.capa_module.Module(etree.tostring(problem), "id").max_score() + response.max_grade = total + response.save() + return (correct, total) @cache_control(no_cache=True, no_store=True, must_revalidate=True) @@ -255,6 +258,7 @@ def profile(request): 'grade_summary' : grade_summary, 'csrf':csrf(request)['csrf_token'] } + return render_to_response('profile.html', context) def format_url_params(params):