From 22748768ab91b297dc3a80d8319ec013f0db9560 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sat, 3 Mar 2012 13:53:24 -0500 Subject: [PATCH 01/20] Cache keys hashed, scripts to assign student to groups and send out bulk e-mails --- courseware/content_parser.py | 10 +-- student/management/__init__.py | 0 student/management/commands/assigngroups.py | 86 +++++++++++++++++++++ student/management/commands/massemail.py | 27 +++++++ 4 files changed, 118 insertions(+), 5 deletions(-) create mode 100644 student/management/__init__.py create mode 100644 student/management/commands/assigngroups.py create mode 100644 student/management/commands/massemail.py diff --git a/courseware/content_parser.py b/courseware/content_parser.py index 0954f60489..190a63fba7 100644 --- a/courseware/content_parser.py +++ b/courseware/content_parser.py @@ -146,11 +146,11 @@ def propogate_downward_tag(element, attribute_name, parent_attribute = None): def user_groups(user): # TODO: Rewrite in Django key = 'user_group_names_{user.id}'.format(user=user) - cache_expiration = 60 * 60 * 4 # four hours - group_names = cache.get(key) + cache_expiration = 60 * 60 # one hour + group_names = cache.get(fasthash(key)) if group_names is None: group_names = [u.name for u in UserTestGroup.objects.filter(users=user)] - cache.set(key, group_names, cache_expiration) + cache.set(fasthash(key), group_names, cache_expiration) return group_names @@ -176,12 +176,12 @@ def course_file(user): cache_key = filename + "_processed?dev_content:" + str(options['dev_content']) + "&groups:" + str(sorted(groups)) - tree_string = cache.get(cache_key) + tree_string = cache.get(fasthash(cache_key)) if not tree_string: tree = course_xml_process(etree.XML(render_to_string(filename, options, namespace = 'course'))) tree_string = etree.tostring(tree) - cache.set(cache_key, tree_string, 60) + cache.set(fasthash(cache_key), tree_string, 60) else: tree = etree.XML(tree_string) diff --git a/student/management/__init__.py b/student/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/student/management/commands/assigngroups.py b/student/management/commands/assigngroups.py new file mode 100644 index 0000000000..150905ab7e --- /dev/null +++ b/student/management/commands/assigngroups.py @@ -0,0 +1,86 @@ +import os.path + +from lxml import etree + +from django.core.management.base import BaseCommand +from django.conf import settings +from django.contrib.auth.models import User + +import mitxmako.middleware as middleware +from student.models import UserTestGroup + +import random +import sys +import datetime + +import json + +middleware.MakoMiddleware() + +def group_from_value(groups, v): + ''' Given group: (('a',0.3),('b',0.4),('c',0.3)) And random value + in [0,1], return the associated group (in the above case, return + 'a' if v<0.3, 'b' if 0.3<=v<0.7, and 'c' if v>0.7 +''' + sum = 0 + for (g,p) in groups: + sum = sum + p + if sum > v: + return g + return g # For round-off errors + +class Command(BaseCommand): + help = \ +''' Assign users to test groups. Takes a list +of groups: +a:0.3,b:0.4,c:0.3 file.txt "Testing something" +Will assign each user to group a, b, or c with +probability 0.3, 0.4, 0.3. Probabilities must +add up to 1. + +Will log what happened to file.txt. +''' + def handle(self, *args, **options): + if len(args) != 3: + print "Invalid number of options" + sys.exit(-1) + + # Extract groups from string + group_strs = [x.split(':') for x in args[0].split(',')] + groups = [(group,float(value)) for group,value in group_strs] + print "Groups", groups + + ## Confirm group probabilities add up to 1 + total = sum(zip(*groups)[1]) + print "Total:", total + if abs(total-1)>0.01: + print "Total not 1" + sys.exit(-1) + + ## Confirm groups don't already exist + for group in dict(groups): + if UserTestGroup.objects.filter(name=group).count() != 0: + print group, "already exists!" + sys.exit(-1) + + group_objects = {} + + ## Create groups + for group in dict(groups): + utg = UserTestGroup() + utg.name=group + utg.description = json.dumps({"description":args[2]}, + {"time":datetime.datetime.utcnow().isoformat()}) + group_objects[group]=utg + group_objects[group].save() + + ## Assign groups + users = list(User.objects.all()) + for user in users: + v = random.uniform(0,1) + group = group_from_value(groups,v) + group_objects[group].users.add(user) + + ## Save groups + for group in group_objects: + group_objects[group].save() diff --git a/student/management/commands/massemail.py b/student/management/commands/massemail.py new file mode 100644 index 0000000000..306ae51c0e --- /dev/null +++ b/student/management/commands/massemail.py @@ -0,0 +1,27 @@ +import os.path + +from lxml import etree + +from django.core.management.base import BaseCommand +from django.conf import settings +from django.contrib.auth.models import User + +import mitxmako.middleware as middleware + +middleware.MakoMiddleware() + +class Command(BaseCommand): + help = \ +'''Sends an e-mail to all users. Takes a single +parameter -- name of e-mail template -- located +in templates/email. Adds a .txt for the message +body, and an _subject.txt for the subject. ''' + def handle(self, *args, **options): + #text = open(args[0]).read() + #subject = open(args[1]).read() + users = User.objects.all() + text = middleware.lookup['main'].get_template('email/'+args[0]+".txt").render() + subject = middleware.lookup['main'].get_template('email/'+args[0]+"_subject.txt").render().strip() + for user in users: + if user.is_active: + user.email_user(subject, text) From 3d4e158782f80a64651629032d06ad308bcbf259 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sat, 3 Mar 2012 14:15:49 -0500 Subject: [PATCH 02/20] Missed on file --- student/management/commands/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 student/management/commands/__init__.py diff --git a/student/management/commands/__init__.py b/student/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From 60312d927bd221ab667fd88a5671d7d44f7aa608 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sat, 3 Mar 2012 14:56:28 -0500 Subject: [PATCH 03/20] switch tracking logs to go to local file --- settings.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/settings.py b/settings.py index cba1535cf3..7eb58af387 100644 --- a/settings.py +++ b/settings.py @@ -174,6 +174,7 @@ execfile(os.path.join(BASE_DIR, "settings.py")) pid = os.getpid() hostname = platform.node().split(".")[0] SYSLOG_ADDRESS = ('syslog.m.i4x.org', 514) +TRACKING_LOG_FILE = LOG_DIR + "/tracking.log" handlers = ['console'] if not DEBUG: @@ -213,6 +214,12 @@ LOGGING = { 'address' : SYSLOG_ADDRESS, 'formatter' : 'syslog_format', }, + 'tracking' : { + 'level' : 'DEBUG', + 'class' : 'logging.handlers.WatchedFileHandler', + 'filename' : TRACKING_LOG_FILE, + 'formatter' : 'raw', + }, 'mail_admins' : { 'level': 'ERROR', 'class': 'django.utils.log.AdminEmailHandler', @@ -225,7 +232,7 @@ LOGGING = { 'level' : 'INFO' }, 'tracking' : { - 'handlers' : [] if DEBUG else ['syslogger'], # handlers, + 'handlers' : ['tracking'], 'level' : 'DEBUG', 'propagate' : False, }, From 04ebd383684ffb7684495e64aed459ce16164dcd Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sat, 3 Mar 2012 15:06:33 -0500 Subject: [PATCH 04/20] update tracking log to use pids --- settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.py b/settings.py index 7eb58af387..61f36465ca 100644 --- a/settings.py +++ b/settings.py @@ -174,7 +174,7 @@ execfile(os.path.join(BASE_DIR, "settings.py")) pid = os.getpid() hostname = platform.node().split(".")[0] SYSLOG_ADDRESS = ('syslog.m.i4x.org', 514) -TRACKING_LOG_FILE = LOG_DIR + "/tracking.log" +TRACKING_LOG_FILE = LOG_DIR + "/tracking_{0}.log".format(pid) handlers = ['console'] if not DEBUG: From 5ae7334b389a292e971e24f493d478d322ee4162 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sat, 3 Mar 2012 15:46:28 -0500 Subject: [PATCH 05/20] Extract email list from DB --- student/management/commands/emaillist.py | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 student/management/commands/emaillist.py diff --git a/student/management/commands/emaillist.py b/student/management/commands/emaillist.py new file mode 100644 index 0000000000..ac4ce33845 --- /dev/null +++ b/student/management/commands/emaillist.py @@ -0,0 +1,26 @@ +import os.path + +from lxml import etree + +from django.core.management.base import BaseCommand +from django.conf import settings +from django.contrib.auth.models import User + +import mitxmako.middleware as middleware + +middleware.MakoMiddleware() + +class Command(BaseCommand): + help = \ +'''Sends an e-mail to all users. Takes a single +parameter -- name of e-mail template -- located +in templates/email. Adds a .txt for the message +body, and an _subject.txt for the subject. ''' + def handle(self, *args, **options): + #text = open(args[0]).read() + #subject = open(args[1]).read() + users = User.objects.all() + + for user in users: + if user.is_active: + print user.email From 385a87c7672727ce8617a683afa5cbc127d83ce1 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sat, 3 Mar 2012 18:02:14 -0500 Subject: [PATCH 06/20] info files now referenced explicitly instead of symlink --- settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.py b/settings.py index 61f36465ca..576c61142f 100644 --- a/settings.py +++ b/settings.py @@ -650,7 +650,7 @@ ot = MAKO_TEMPLATES MAKO_TEMPLATES['course'] = [DATA_DIR] MAKO_TEMPLATES['sections'] = [DATA_DIR+'/sections'] MAKO_TEMPLATES['custom_tags'] = [DATA_DIR+'/custom_tags'] -MAKO_TEMPLATES['main'] = [BASE_DIR+'/templates/'] +MAKO_TEMPLATES['main'] = [BASE_DIR+'/templates/', DATA_DIR+'/info'] MAKO_TEMPLATES.update(ot) From 3a6bb0cd8bc93a54c6ce7a386de3a31305fae008 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sat, 3 Mar 2012 19:21:03 -0500 Subject: [PATCH 07/20] Properly logging --- student/management/commands/assigngroups.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/student/management/commands/assigngroups.py b/student/management/commands/assigngroups.py index 150905ab7e..87c15bb1ab 100644 --- a/student/management/commands/assigngroups.py +++ b/student/management/commands/assigngroups.py @@ -65,6 +65,8 @@ Will log what happened to file.txt. group_objects = {} + f = open(args[1],"a+") + ## Create groups for group in dict(groups): utg = UserTestGroup() @@ -76,11 +78,22 @@ Will log what happened to file.txt. ## Assign groups users = list(User.objects.all()) + count = 0 for user in users: + if count % 1000 == 0: + print count + count = count + 1 v = random.uniform(0,1) group = group_from_value(groups,v) group_objects[group].users.add(user) + f.write("Assigned user {name} ({id}) to {group}\n".format(name=user.username, + id=user.id, + group=group)) ## Save groups for group in group_objects: group_objects[group].save() + f.close() + +# python manage.py assigngroups summary_test:0.3,skip_summary_test:0.7 log.txt "Do previews of future materials help?" +# python manage.py assigngroups skip_capacitor:0.3,capacitor:0.7 log.txt "Do we show capacitor in linearity tutorial?" From cb24b172794a378bd37dbcd1e34b7e58f054b561 Mon Sep 17 00:00:00 2001 From: Bridger Maxwell Date: Sat, 3 Mar 2012 23:49:51 -0500 Subject: [PATCH 08/20] Changed the wiki titles to say 6.002x (they were missing the x) --- simplewiki/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplewiki/views.py b/simplewiki/views.py index 4766b4f270..e2324c8e3b 100644 --- a/simplewiki/views.py +++ b/simplewiki/views.py @@ -39,7 +39,7 @@ def view(request, wiki_url): 'wiki_write': article.can_write_l(request.user), 'wiki_attachments_write': article.can_attach(request.user), 'wiki_current_revision_deleted' : not (article.current_revision.deleted == 0), - 'wiki_title' : article.title + " - MITX 6.002 Wiki" + 'wiki_title' : article.title + " - MITX 6.002x Wiki" } d.update(csrf(request)) return render_to_response('simplewiki_view.html', d) From 98f5a79dfc3c5bd2b3595020490b03b976f02bb1 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 4 Mar 2012 09:40:07 -0500 Subject: [PATCH 09/20] Disabled profile cache -- we ran into issues --- cache_toolbox/core.py | 5 +++++ cache_toolbox/relation.py | 4 ++++ courseware/content_parser.py | 6 +++++- courseware/views.py | 6 +++++- student/models.py | 2 +- student/views.py | 2 +- 6 files changed, 21 insertions(+), 4 deletions(-) diff --git a/cache_toolbox/core.py b/cache_toolbox/core.py index 8abc16c7be..b2802e7fec 100644 --- a/cache_toolbox/core.py +++ b/cache_toolbox/core.py @@ -58,6 +58,11 @@ def get_instance(model, instance_or_pk, timeout=None, using=None): cache.delete(key) # Use the default manager so we are never filtered by a .get_query_set() + +# import logging +# log = logging.getLogger("tracking") +# log.info( str(pk) ) + instance = model._default_manager.using(using).get(pk=pk) data = {} diff --git a/cache_toolbox/relation.py b/cache_toolbox/relation.py index eae7d93f6f..38d985aa94 100644 --- a/cache_toolbox/relation.py +++ b/cache_toolbox/relation.py @@ -92,6 +92,10 @@ def cache_relation(descriptor, timeout=None): except AttributeError: pass +# import logging +# log = logging.getLogger("tracking") +# log.info( "DEBUG: "+str(str(rel.model)+"/"+str(self.pk) )) + instance = get_instance(rel.model, self.pk, timeout) setattr(self, '_%s_cache' % related_name, instance) diff --git a/courseware/content_parser.py b/courseware/content_parser.py index 190a63fba7..30b8c1753b 100644 --- a/courseware/content_parser.py +++ b/courseware/content_parser.py @@ -169,7 +169,11 @@ def course_xml_process(tree): def course_file(user): ''' Given a user, return course.xml''' - filename = user.profile_cache.courseware # UserProfile.objects.get(user=user).courseware + #import logging + #log = logging.getLogger("tracking") + #log.info( "DEBUG: cf:"+str(user) ) + + filename = UserProfile.objects.get(user=user).courseware # user.profile_cache.courseware groups = user_groups(user) options = {'dev_content':settings.DEV_CONTENT, 'groups' : groups} diff --git a/courseware/views.py b/courseware/views.py index 3a1835e1bd..876c0563e1 100644 --- a/courseware/views.py +++ b/courseware/views.py @@ -254,7 +254,7 @@ def profile(request): ] - user_info = request.user.profile_cache # UserProfile.objects.get(user=request.user) + user_info = UserProfile.objects.get(user=request.user) # request.user.profile_cache # context={'name':user_info.name, 'username':request.user.username, 'location':user_info.location, @@ -345,6 +345,10 @@ def index(request, course="6.002 Spring 2012", chapter="Using the System", secti if course!="6.002 Spring 2012": return redirect('/') + #import logging + #log = logging.getLogger("mitx") + #log.info( "DEBUG: "+str(user) ) + dom = content_parser.course_file(user) dom_module = dom.xpath("//course[@name=$course]/chapter[@name=$chapter]//section[@name=$section]/*[1]", course=course, chapter=chapter, section=section) diff --git a/student/models.py b/student/models.py index 8c9a71c753..d12d2e4aae 100644 --- a/student/models.py +++ b/student/models.py @@ -56,4 +56,4 @@ class Registration(models.Model): self.user.save() #self.delete() -cache_relation(User.profile) +#cache_relation(User.profile) diff --git a/student/views.py b/student/views.py index 5e3cb7b9b0..b8194b3e61 100644 --- a/student/views.py +++ b/student/views.py @@ -86,7 +86,7 @@ def logout_user(request): def change_setting(request): if not request.user.is_authenticated(): return redirect('/') - up = request.user.profile_cache # UserProfile.objects.get(user=request.user) + up = UserProfile.objects.get(user=request.user) #request.user.profile_cache if 'location' in request.POST: # print "loc" up.location=request.POST['location'] From b348f5d9b3781339ed7506ed153ace6093b01738 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 4 Mar 2012 13:25:12 -0500 Subject: [PATCH 10/20] Cache toolbox disabled --- courseware/models.py | 46 ++++++++++++++++++++++---------------------- student/models.py | 2 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/courseware/models.py b/courseware/models.py index 24c212366a..a97b09ae2b 100644 --- a/courseware/models.py +++ b/courseware/models.py @@ -14,12 +14,12 @@ ASSUMPTIONS: modules have unique IDs, even across different module_types """ from django.db import models from django.db.models.signals import post_save, post_delete -from django.core.cache import cache +#from django.core.cache import cache from django.contrib.auth.models import User -from cache_toolbox import cache_model, cache_relation +#from cache_toolbox import cache_model, cache_relation -CACHE_TIMEOUT = 60 * 60 * 4 # Set the cache timeout to be four hours +#CACHE_TIMEOUT = 60 * 60 * 4 # Set the cache timeout to be four hours class StudentModule(models.Model): # For a homework problem, contains a JSON @@ -58,35 +58,35 @@ class StudentModule(models.Model): def __unicode__(self): return self.module_type+'/'+self.student.username+"/"+self.module_id+'/'+str(self.state)[:20] - @classmethod - def get_with_caching(cls, student, module_id): - k = cls.key_for(student, module_id) - student_module = cache.get(k) - if student_module is None: - student_module = StudentModule.objects.filter(student=student, - module_id=module_id)[0] - # It's possible it really doesn't exist... - if student_module is not None: - cache.set(k, student_module, CACHE_TIMEOUT) + # @classmethod + # def get_with_caching(cls, student, module_id): + # k = cls.key_for(student, module_id) + # student_module = cache.get(k) + # if student_module is None: + # student_module = StudentModule.objects.filter(student=student, + # module_id=module_id)[0] + # # It's possible it really doesn't exist... + # if student_module is not None: + # cache.set(k, student_module, CACHE_TIMEOUT) - return student_module + # return student_module @classmethod def key_for(cls, student, module_id): return "StudentModule-student_id:{0};module_id:{1}".format(student.id, module_id) -def clear_cache_by_student_and_module_id(sender, instance, *args, **kwargs): - k = sender.key_for(instance.student, instance.module_id) - cache.delete(k) +# def clear_cache_by_student_and_module_id(sender, instance, *args, **kwargs): +# k = sender.key_for(instance.student, instance.module_id) +# cache.delete(k) -def update_cache_by_student_and_module_id(sender, instance, *args, **kwargs): - k = sender.key_for(instance.student, instance.module_id) - cache.set(k, instance, CACHE_TIMEOUT) +# def update_cache_by_student_and_module_id(sender, instance, *args, **kwargs): +# k = sender.key_for(instance.student, instance.module_id) +# cache.set(k, instance, CACHE_TIMEOUT) -post_save.connect(update_cache_by_student_and_module_id, sender=StudentModule, weak=False) -post_delete.connect(clear_cache_by_student_and_module_id, sender=StudentModule, weak=False) +#post_save.connect(update_cache_by_student_and_module_id, sender=StudentModule, weak=False) +#post_delete.connect(clear_cache_by_student_and_module_id, sender=StudentModule, weak=False) -cache_model(StudentModule) +#cache_model(StudentModule) diff --git a/student/models.py b/student/models.py index d12d2e4aae..b26041a4aa 100644 --- a/student/models.py +++ b/student/models.py @@ -13,7 +13,7 @@ import uuid from django.db import models from django.contrib.auth.models import User -from cache_toolbox import cache_model, cache_relation +#from cache_toolbox import cache_model, cache_relation class UserProfile(models.Model): class Meta: From d1276e9af7fb922a133f19adb6d3c1f341356863 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 4 Mar 2012 14:06:08 -0500 Subject: [PATCH 11/20] Missed a spot --- courseware/module_render.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/courseware/module_render.py b/courseware/module_render.py index 59660ad687..82fafa3234 100644 --- a/courseware/module_render.py +++ b/courseware/module_render.py @@ -48,9 +48,9 @@ def make_track_function(request): def modx_dispatch(request, module=None, dispatch=None, id=None): ''' Generic view for extensions. ''' # Grab the student information for the module from the database - #s = StudentModule.objects.filter(student=request.user, - # module_id=id) - s = StudentModule.get_with_caching(request.user, id) + s = StudentModule.objects.filter(student=request.user, + module_id=id) + #s = StudentModule.get_with_caching(request.user, id) if s is None: log.debug("Couldnt find module for user and id " + str(module) + " " + str(request.user) + " "+ str(id)) raise Http404 From f27c16c87a62f8e230cb8e82ba049bd19c11cfa9 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 4 Mar 2012 14:09:14 -0500 Subject: [PATCH 12/20] Missed another spot --- courseware/module_render.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/courseware/module_render.py b/courseware/module_render.py index 82fafa3234..cef037be70 100644 --- a/courseware/module_render.py +++ b/courseware/module_render.py @@ -51,9 +51,10 @@ def modx_dispatch(request, module=None, dispatch=None, id=None): s = StudentModule.objects.filter(student=request.user, module_id=id) #s = StudentModule.get_with_caching(request.user, id) - if s is None: + if len(s) == 0 or s is None: log.debug("Couldnt find module for user and id " + str(module) + " " + str(request.user) + " "+ str(id)) raise Http404 + s = s[0] oldgrade = s.grade oldstate = s.state From cc13aef05a7221ed0d237922413718f9d613e907 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 4 Mar 2012 14:21:52 -0500 Subject: [PATCH 13/20] Check on user authenticated in AJAX --- courseware/module_render.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/courseware/module_render.py b/courseware/module_render.py index cef037be70..d1f1f89930 100644 --- a/courseware/module_render.py +++ b/courseware/module_render.py @@ -47,6 +47,9 @@ def make_track_function(request): def modx_dispatch(request, module=None, dispatch=None, id=None): ''' Generic view for extensions. ''' + if not request.user.is_authenticated(): + return redirect('/') + # Grab the student information for the module from the database s = StudentModule.objects.filter(student=request.user, module_id=id) From fe082d62484558c74ce3eec0e8380ad54c06d2db Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Mon, 5 Mar 2012 08:55:18 -0500 Subject: [PATCH 14/20] Script to e-mail users in bulk, in batches, from a text file --- student/management/commands/emaillist.py | 5 +- student/management/commands/massemailtxt.py | 53 +++++++++++++++++++++ 2 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 student/management/commands/massemailtxt.py diff --git a/student/management/commands/emaillist.py b/student/management/commands/emaillist.py index ac4ce33845..019e98aac5 100644 --- a/student/management/commands/emaillist.py +++ b/student/management/commands/emaillist.py @@ -12,10 +12,7 @@ middleware.MakoMiddleware() class Command(BaseCommand): help = \ -'''Sends an e-mail to all users. Takes a single -parameter -- name of e-mail template -- located -in templates/email. Adds a .txt for the message -body, and an _subject.txt for the subject. ''' +''' Extract an e-mail list of all active students. ''' def handle(self, *args, **options): #text = open(args[0]).read() #subject = open(args[1]).read() diff --git a/student/management/commands/massemailtxt.py b/student/management/commands/massemailtxt.py new file mode 100644 index 0000000000..4d9f32b282 --- /dev/null +++ b/student/management/commands/massemailtxt.py @@ -0,0 +1,53 @@ +import os.path +import time + +from lxml import etree + +from django.core.management.base import BaseCommand +from django.conf import settings +from django.contrib.auth.models import User + +import mitxmako.middleware as middleware + +from django.core.mail import send_mass_mail + +middleware.MakoMiddleware() + +def chunks(l, n): + """ Yield successive n-sized chunks from l. + """ + for i in xrange(0, len(l), n): + yield l[i:i+n] + +class Command(BaseCommand): + help = \ +'''Sends an e-mail to all users in a text file. +E.g. +manage.py userlist.txt message logfile.txt rate +userlist.txt -- list of all users +message -- prefix for template with message +logfile.txt -- where to log progress +rate -- messages per second + ''' + log_file = None + + def hard_log(self, text): + self.log_file.write(text+'\n') + + def handle(self, *args, **options): + global log_file + (user_file, message_base, logfilename, ratestr) = args + + users = [u.strip() for u in open(user_file).readlines()] + + message = middleware.lookup['main'].get_template('emails/'+message_base+"_body.txt").render() + subject = middleware.lookup['main'].get_template('emails/'+message_base+"_subject.txt").render().strip() + rate = int(ratestr) + + self.log_file = open(logfilename, "a+", buffering = 0) + + for users in chunks(users, rate): + emails = [ (subject, message, settings.DEFAULT_FROM_EMAIL, [u]) for u in users ] + self.hard_log(" ".join(users)) + send_mass_mail( emails, fail_silently = False ) + time.sleep(1) From 4325437c31180963eb8cf6ad68ccb5d71078d350 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Mon, 5 Mar 2012 09:04:23 -0500 Subject: [PATCH 15/20] Timestamps in log file for bulk emails --- student/management/commands/massemailtxt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/student/management/commands/massemailtxt.py b/student/management/commands/massemailtxt.py index 4d9f32b282..f29216dec1 100644 --- a/student/management/commands/massemailtxt.py +++ b/student/management/commands/massemailtxt.py @@ -11,6 +11,8 @@ import mitxmako.middleware as middleware from django.core.mail import send_mass_mail +import datetime + middleware.MakoMiddleware() def chunks(l, n): @@ -32,7 +34,7 @@ rate -- messages per second log_file = None def hard_log(self, text): - self.log_file.write(text+'\n') + self.log_file.write(datetime.datetime.utcnow().isoformat()+' -- '+text+'\n') def handle(self, *args, **options): global log_file From a379ce365fe02522f8676f467d4259ed3a4bffb0 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Mon, 5 Mar 2012 09:09:21 -0500 Subject: [PATCH 16/20] Give number of e-mails sent --- student/management/commands/massemailtxt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/student/management/commands/massemailtxt.py b/student/management/commands/massemailtxt.py index f29216dec1..d72de86c50 100644 --- a/student/management/commands/massemailtxt.py +++ b/student/management/commands/massemailtxt.py @@ -48,8 +48,11 @@ rate -- messages per second self.log_file = open(logfilename, "a+", buffering = 0) + i=0 for users in chunks(users, rate): emails = [ (subject, message, settings.DEFAULT_FROM_EMAIL, [u]) for u in users ] self.hard_log(" ".join(users)) send_mass_mail( emails, fail_silently = False ) time.sleep(1) + print i, + i = i+len(users) From 7d6a0af1154f191170bee3688b259241b835b29a Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Mon, 5 Mar 2012 09:23:04 -0500 Subject: [PATCH 17/20] Emergency fallback if we need to stop the presses on bulk emails --- student/management/commands/massemailtxt.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/student/management/commands/massemailtxt.py b/student/management/commands/massemailtxt.py index d72de86c50..73f06b87a0 100644 --- a/student/management/commands/massemailtxt.py +++ b/student/management/commands/massemailtxt.py @@ -10,6 +10,7 @@ from django.contrib.auth.models import User import mitxmako.middleware as middleware from django.core.mail import send_mass_mail +import sys import datetime @@ -56,3 +57,8 @@ rate -- messages per second time.sleep(1) print i, i = i+len(users) + # Emergency interruptor + if os.path.exists("/tmp/stopemails.txt"): + self.log_file.close() + sys.exit(-1) + self.log_file.close() From 9e42554c2b62d2ed33012945884c49e6856f7f4f Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Mon, 5 Mar 2012 09:33:56 -0500 Subject: [PATCH 18/20] Better status when sending e-mails --- student/management/commands/massemailtxt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/student/management/commands/massemailtxt.py b/student/management/commands/massemailtxt.py index 73f06b87a0..661c4e4fb7 100644 --- a/student/management/commands/massemailtxt.py +++ b/student/management/commands/massemailtxt.py @@ -55,7 +55,7 @@ rate -- messages per second self.hard_log(" ".join(users)) send_mass_mail( emails, fail_silently = False ) time.sleep(1) - print i, + print datetime.datetime.utcnow().isoformat(), i i = i+len(users) # Emergency interruptor if os.path.exists("/tmp/stopemails.txt"): From 299e1ae905e5526f752efb4fc086b4bcdc913890 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Tue, 6 Mar 2012 10:55:45 -0500 Subject: [PATCH 19/20] fixed it so staff can immediately accept answers --- settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.py b/settings.py index 576c61142f..decfa77580 100644 --- a/settings.py +++ b/settings.py @@ -631,7 +631,7 @@ LIVESETTINGS_OPTIONS = { 'VOTE_RULES' : { 'MAX_VOTES_PER_USER_PER_DAY' : 30, 'MAX_FLAGS_PER_USER_PER_DAY' : 5, - 'MIN_DAYS_FOR_STAFF_TO_ACCEPT_ANSWER' : 7, + 'MIN_DAYS_FOR_STAFF_TO_ACCEPT_ANSWER' : 0, 'MIN_DAYS_TO_ANSWER_OWN_QUESTION' : 0, 'MIN_FLAGS_TO_DELETE_POST' : 5, 'MIN_FLAGS_TO_HIDE_POST' : 3, From ccacc90ca7ce0c440c5113a80ed572e95c8b104b Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Tue, 6 Mar 2012 11:44:26 -0500 Subject: [PATCH 20/20] Don't allow posting before logging in. --- settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.py b/settings.py index decfa77580..e8b0d2b1e7 100644 --- a/settings.py +++ b/settings.py @@ -446,7 +446,7 @@ LIVESETTINGS_OPTIONS = { 'MIN_ANSWER_BODY_LENGTH' : 1, 'WIKI_ON' : True, 'ALLOW_ASK_ANONYMOUSLY' : True, - 'ALLOW_POSTING_BEFORE_LOGGING_IN' : True, + 'ALLOW_POSTING_BEFORE_LOGGING_IN' : False, 'ALLOW_SWAPPING_QUESTION_WITH_ANSWER' : False, 'MAX_TAG_LENGTH' : 20, 'MIN_TITLE_LENGTH' : 1,