Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Get the cms up to the point of rendering a template

Browse Source
This commit is contained in:
Calen Pennington
2012-06-08 14:01:41 -04:00
parent 05436e591b
commit 8134bcbc3d
35 changed files with 240 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
lms/djangoapps/track/__init__.py
View File

24
lms/djangoapps/track/middleware.py
View File

@@ -1,24 +0,0 @@
import json
from django.conf import settings
import views
class TrackMiddleware:
def process_request(self, request):
try:
# We're already logging events, and we don't want to capture user
# names/passwords.
if request.META['PATH_INFO'] in ['/event', '/login']:
return
event = { 'GET' : dict(request.GET),
'POST' : dict(request.POST)}
# TODO: Confirm no large file uploads
event = json.dumps(event)
event = event[:512]
views.server_track(request, request.META['PATH_INFO'], event)
except:
pass

3
lms/djangoapps/track/models.py
View File

@@ -1,3 +0,0 @@
from django.db import models
# Create your models here.

16
lms/djangoapps/track/tests.py
View File

@@ -1,16 +0,0 @@
"""
This file demonstrates writing tests using the unittest module. These will pass
when you run "manage.py test".
Replace this with more appropriate tests for your application.
"""
from django.test import TestCase
class SimpleTest(TestCase):
def test_basic_addition(self):
"""
Tests that 1 + 1 always equals 2.
"""
self.assertEqual(1 + 1, 2)

70
lms/djangoapps/track/views.py
View File

@@ -1,70 +0,0 @@
import json
import logging
import os
import datetime
# Create your views here.
from django.http import HttpResponse
from django.http import Http404
from django.conf import settings
log = logging.getLogger("tracking")
def log_event(event):
event_str = json.dumps(event)
log.info(event_str[:settings.TRACK_MAX_EVENT])
def user_track(request):
try: # TODO: Do the same for many of the optional META parameters
username = request.user.username
except:
username = "anonymous"
try:
scookie = request.META['HTTP_COOKIE'] # Get cookies
scookie = ";".join([c.split('=')[1] for c in scookie.split(";") if "sessionid" in c]).strip() # Extract session ID
except:
scookie = ""
try:
agent = request.META['HTTP_USER_AGENT']
except:
agent = ''
# TODO: Move a bunch of this into log_event
event = {
"username" : username,
"session" : scookie,
"ip" : request.META['REMOTE_ADDR'],
"event_source" : "browser",
"event_type" : request.GET['event_type'],
"event" : request.GET['event'],
"agent" : agent,
"page" : request.GET['page'],
"time": datetime.datetime.utcnow().isoformat(),
}
log_event(event)
return HttpResponse('success')
def server_track(request, event_type, event, page=None):
try:
username = request.user.username
except:
username = "anonymous"
try:
agent = request.META['HTTP_USER_AGENT']
except:
agent = ''
event = {
"username" : username,
"ip" : request.META['REMOTE_ADDR'],
"event_source" : "server",
"event_type" : event_type,
"event" : event,
"agent" : agent,
"page" : page,
"time": datetime.datetime.utcnow().isoformat(),
}
log_event(event)

1
lms/envs/common.py
View File

@@ -60,6 +60,7 @@ sys.path.append(ASKBOT_ROOT)
sys.path.append(ASKBOT_ROOT / "askbot" / "deps")
sys.path.append(PROJECT_ROOT / 'djangoapps')
sys.path.append(PROJECT_ROOT / 'lib')
sys.path.append(COMMON_ROOT / 'djangoapps')
sys.path.append(COMMON_ROOT / 'lib')
################################## MITXWEB #####################################

24
lms/lib/cache_toolbox/COPYING
View File

@@ -1,24 +0,0 @@
Copyright © 2010, 2011 UUMC Ltd. <tech@playfire.com>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of the <organization> nor the
names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

4
lms/lib/cache_toolbox/README.rst
View File

@@ -1,4 +0,0 @@
django-cache-toolbox
============================
Documentation: http://code.playfire.com/django-cache-toolbox/

29
lms/lib/cache_toolbox/__init__.py
View File

@@ -1,29 +0,0 @@
"""
:mod:`cache_toolbox` --- Non-magical object caching tools for Django
====================================================================
Introduction
------------
``cache_toolbox`` is intended to be a lightweight series of independent tools
to leverage caching within Django projects.
The tools are deliberately `non-magical`. That is to say, instances are never
retrieved from caches behind your back and regular Django ``.filter()`` /
``.get()`` queries continue to work exactly as before.
Because of this, you can introduce ``cache_toolbox`` into your project slowly
when needed rather than "switching" to it with invasive changes.
Links
-----
View/download code
https://github.com/playfire/django-cache-toolbox
File a bug
https://github.com/playfire/django-cache-toolbox/issues
"""
from .model import cache_model
from .relation import cache_relation

8
lms/lib/cache_toolbox/app_settings.py
View File

@@ -1,8 +0,0 @@
from django.conf import settings
# Default cache timeout
CACHE_TOOLBOX_DEFAULT_TIMEOUT = getattr(
settings,
'CACHE_TOOLBOX_DEFAULT_TIMEOUT',
60 * 60 * 24 * 3,
)

106
lms/lib/cache_toolbox/core.py
View File

@@ -1,106 +0,0 @@
"""
Core methods
------------
.. autofunction:: cache_toolbox.core.get_instance
.. autofunction:: cache_toolbox.core.delete_instance
.. autofunction:: cache_toolbox.core.instance_key
"""
from django.core.cache import cache
from django.db import DEFAULT_DB_ALIAS
from . import app_settings
def get_instance(model, instance_or_pk, timeout=None, using=None):
"""
Returns the ``model`` instance with a primary key of ``instance_or_pk``.
If the data is cached it will be returned from there, otherwise the regular
Django ORM is queried for this instance and the data stored in the cache.
If omitted, the timeout value defaults to
``settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT`` instead of 0 (zero).
Example::
>>> get_instance(User, 1) # Cache miss
<User: lamby>
>>> get_instance(User, 1) # Cache hit
<User: lamby>
>>> User.objects.get(pk=1) == get_instance(User, 1)
True
"""
pk = getattr(instance_or_pk, 'pk', instance_or_pk)
key = instance_key(model, instance_or_pk)
data = cache.get(key)
if data is not None:
try:
# Try and construct instance from dictionary
instance = model(pk=pk, **data)
# Ensure instance knows that it already exists in the database,
# otherwise we will fail any uniqueness checks when saving the
# instance.
instance._state.adding = False
# Specify database so that instance is setup correctly. We don't
# namespace cached objects by their origin database, however.
instance._state.db = using or DEFAULT_DB_ALIAS
return instance
except:
# Error when deserialising - remove from the cache; we will
# fallback and return the underlying instance
cache.delete(key)
# Use the default manager so we are never filtered by a .get_query_set()
# import logging
# log = logging.getLogger("tracking")
# log.info( str(pk) )
instance = model._default_manager.using(using).get(pk=pk)
data = {}
for field in instance._meta.fields:
# Harmless to save, but saves space in the dictionary - we already know
# the primary key when we lookup
if field.primary_key:
continue
if field.get_internal_type() == 'FileField':
# Avoid problems with serializing FileFields
# by only serializing the file name
file = getattr(instance, field.attname)
data[field.attname] = file.name
else:
data[field.attname] = getattr(instance, field.attname)
if timeout is None:
timeout = app_settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT
cache.set(key, data, timeout)
return instance
def delete_instance(model, *instance_or_pk):
"""
Purges the cache keys for the instances of this model.
"""
cache.delete_many([instance_key(model, x) for x in instance_or_pk])
def instance_key(model, instance_or_pk):
"""
Returns the cache key for this (model, instance) pair.
"""
return '%s.%s:%d' % (
model._meta.app_label,
model._meta.module_name,
getattr(instance_or_pk, 'pk', instance_or_pk),
)

97
lms/lib/cache_toolbox/middleware.py
View File

@@ -1,97 +0,0 @@
"""
Cache-backed ``AuthenticationMiddleware``
-----------------------------------------
``CacheBackedAuthenticationMiddleware`` is an
``django.contrib.auth.middleware.AuthenticationMiddleware`` replacement to
avoid querying the database for a ``User`` instance in each request.
Whilst the built-in ``AuthenticationMiddleware`` mechanism will only obtain the
``User`` instance when it is required, the vast majority of sites will do so on
every page to render "Logged in as 'X'" text as well to evaluate the result of
``user.is_authenticated()`` and ``user.is_superuser`` to provide conditional
functionality.
This middleware eliminates the cost of retrieving this ``User`` instance by
caching it using the ``cache_toolbox`` instance caching mechanisms.
Depending on your average number of queries per page, saving one query per
request can---in aggregate---reduce load on your database. In addition,
avoiding the database entirely for pages can avoid incurring any connection
latency in your environment, resulting in faster page loads for your users.
Saving this data in the cache can also be used as a way of authenticating users
in systems outside of Django that should not access your database. For
example, a "maintenance mode" page would be able to render a personalised
message without touching the database at all but rather authenticating via the
cache.
``CacheBackedAuthenticationMiddleware`` is ``AUTHENTICATION_BACKENDS`` agnostic.
Implementation
~~~~~~~~~~~~~~
The cache and session backends are still accessed on each request - we are
simply assuming that they are cheaper (or otherwise more preferable) to access
than your database. (In the future, signed cookies may allow us to avoid this
lookup altogether -- whilst we could not safely save ``User.password`` in a
cookie, we could use delayed loading to pull it out when needed.)
Another alternative solution would be to store the attributes in the user's
session instead of in the cache. This would save the cache hit on every request
as all the relevant data would be pulled in one go from the session backend.
However, this has two main disadvantages:
* Session keys are not deterministic -- after making changes to an
``auth_user`` row in the database, you cannot determine the user's session
key to flush the now out-of-sync data (and doing so would log them out
anyway).
* Stores data per-session rather than per-user -- if a user logs in from
multiple computers the data is duplicated in each session. This problem is
compounded by most projects wishing to avoid expiring session data as long
as possible (in addition to storing sessions in persistent stores).
Usage
~~~~~
To use, find ``MIDDLEWARE_CLASSES`` in your ``settings.py`` and replace::
MIDDLEWARE_CLASSES = [
...
'django.contrib.auth.middleware.AuthenticationMiddleware',
...
]
with::
MIDDLEWARE_CLASSES = [
...
'cache_toolbox.middleware.CacheBackedAuthenticationMiddleware',
...
]
You should confirm you are using a ``SESSION_ENGINE`` that doesn't query the
database for each request. The built-in ``cached_db`` engine is the safest
choice for most environments but you may be happy with the trade-offs of the
``memcached`` backend - see the Django documentation for more details.
"""
from django.contrib.auth import SESSION_KEY
from django.contrib.auth.models import User
from django.contrib.auth.middleware import AuthenticationMiddleware
from .model import cache_model
class CacheBackedAuthenticationMiddleware(AuthenticationMiddleware):
def __init__(self):
cache_model(User)
def process_request(self, request):
try:
# Try and construct a User instance from data stored in the cache
request.user = User.get_cached(request.session[SESSION_KEY])
except:
# Fallback to constructing the User from the database.
super(CacheBackedAuthenticationMiddleware, self).process_request(request)

78
lms/lib/cache_toolbox/model.py
View File

@@ -1,78 +0,0 @@
"""
Caching model instances
-----------------------
``cache_model`` adds utility methods to a model to obtain ``ForeignKey``
instances via the cache.
Usage
~~~~~
::
from django.db import models
from django.contrib.auth.models import User
class Foo(models.Model):
name = models.CharField(length=20)
cache_model(Foo)
::
>>> a = Foo.objects.create(name='a')
>>> a
<Foo: >
>>> Foo.get_cached(a.pk) # Cache miss
<Foo: >
>>> a = Foo.get_cached(a.pk) # Cache hit
>>> a.name
u'a'
Instances returned from ``get_cached`` are real model instances::
>>> a = Foo.get_cached(a.pk) # Cache hit
>>> type(a)
<class '__main__.models.A'>
>>> a.pk
1L
Invalidation
~~~~~~~~~~~~
Invalidation is performed automatically upon saving or deleting a ``Foo``
instance::
>>> a = Foo.objects.create(name='a')
>>> a.name = 'b'
>>> a.save()
>>> a = Foo.get_cached(a.pk)
>>> a.name
u'b'
>>> a.delete()
>>> a = Foo.get_cached(a.pk)
... Foo.DoesNotExist
"""
from django.db.models.signals import post_save, post_delete
from .core import get_instance, delete_instance
def cache_model(model, timeout=None):
if hasattr(model, 'get_cached'):
# Already patched
return
def clear_cache(sender, instance, *args, **kwargs):
delete_instance(sender, instance)
post_save.connect(clear_cache, sender=model, weak=False)
post_delete.connect(clear_cache, sender=model, weak=False)
@classmethod
def get(cls, pk, using=None):
if pk is None:
return None
return get_instance(cls, pk, timeout, using)
model.get_cached = get

122
lms/lib/cache_toolbox/relation.py
View File

@@ -1,122 +0,0 @@
"""
Caching instances via ``related_name``
--------------------------------------
``cache_relation`` adds utility methods to a model to obtain ``related_name``
instances via the cache.
Usage
~~~~~
::
from django.db import models
from django.contrib.auth.models import User
class Foo(models.Model):
user = models.OneToOneField(
User,
primary_key=True,
related_name='foo',
)
name = models.CharField(max_length=20)
cache_relation(User.foo)
::
>>> user = User.objects.get(pk=1)
>>> user.foo_cache # Cache miss - hits the database
<Foo: >
>>> user = User.objects.get(pk=1)
>>> user.foo_cache # Cache hit - no database access
<Foo: >
>>> user = User.objects.get(pk=2)
>>> user.foo # Regular lookup - hits the database
<Foo: >
>>> user.foo_cache # Special-case: Will not hit cache or database.
<Foo: >
Accessing ``user_instance.foo_cache`` (note the "_cache" suffix) will now
obtain the related ``Foo`` instance via the cache. Accessing the original
``user_instance.foo`` attribute will perform the lookup as normal.
Invalidation
~~~~~~~~~~~~
Upon saving (or deleting) the instance, the cache is cleared. For example::
>>> user = User.objects.get(pk=1)
>>> foo = user.foo_cache # (Assume cache hit from previous session)
>>> foo.name = "New name"
>>> foo.save() # Cache is cleared on save
>>> user = User.objects.get(pk=1)
>>> user.foo_cache # Cache miss.
<Foo: >
Manual invalidation may also be performed using the following methods::
>>> user_instance.foo_cache_clear()
>>> User.foo_cache_clear_fk(user_instance_pk)
Manual invalidation is required if you use ``.update()`` methods which the
``post_save`` and ``post_delete`` hooks cannot intercept.
Support
~~~~~~~
``cache_relation`` currently only works with ``OneToOneField`` fields. Support
for regular ``ForeignKey`` fields is planned.
"""
from django.db.models.signals import post_save, post_delete
from .core import get_instance, delete_instance
def cache_relation(descriptor, timeout=None):
rel = descriptor.related
related_name = '%s_cache' % rel.field.related_query_name()
@property
def get(self):
# Always use the cached "real" instance if available
try:
return getattr(self, descriptor.cache_name)
except AttributeError:
pass
# Lookup cached instance
try:
return getattr(self, '_%s_cache' % related_name)
except AttributeError:
pass
# import logging
# log = logging.getLogger("tracking")
# log.info( "DEBUG: "+str(str(rel.model)+"/"+str(self.pk) ))
instance = get_instance(rel.model, self.pk, timeout)
setattr(self, '_%s_cache' % related_name, instance)
return instance
setattr(rel.parent_model, related_name, get)
# Clearing cache
def clear(self):
delete_instance(rel.model, self)
@classmethod
def clear_pk(cls, *instances_or_pk):
delete_instance(rel.model, *instances_or_pk)
def clear_cache(sender, instance, *args, **kwargs):
delete_instance(rel.model, instance)
setattr(rel.parent_model, '%s_clear' % related_name, clear)
setattr(rel.parent_model, '%s_clear_pk' % related_name, clear_pk)
post_save.connect(clear_cache, sender=rel.model, weak=False)
post_delete.connect(clear_cache, sender=rel.model, weak=False)

0
lms/lib/cache_toolbox/templatetags/__init__.py
View File

69
lms/lib/cache_toolbox/templatetags/cache_toolbox.py
View File

@@ -1,69 +0,0 @@
from django import template
from django.core.cache import cache
from django.template import Node, TemplateSyntaxError, Variable
from django.template import resolve_variable
register = template.Library()
class CacheNode(Node):
def __init__(self, nodelist, expire_time, key):
self.nodelist = nodelist
self.expire_time = Variable(expire_time)
self.key = key
def render(self, context):
key = resolve_variable(self.key, context)
expire_time = int(self.expire_time.resolve(context))
value = cache.get(key)
if value is None:
value = self.nodelist.render(context)
cache.set(key, value, expire_time)
return value
@register.tag
def cachedeterministic(parser, token):
"""
This will cache the contents of a template fragment for a given amount of
time, just like {% cache .. %} except that the key is deterministic and not
mangled or run through MD5.
Usage::
{% cachedeterministic [expire_time] [key] %}
.. some expensive processing ..
{% endcachedeterministic %}
"""
nodelist = parser.parse(('endcachedeterministic',))
parser.delete_first_token()
tokens = token.contents.split()
if len(tokens) != 3:
raise TemplateSyntaxError(u"'%r' tag requires 2 arguments." % tokens[0])
return CacheNode(nodelist, tokens[1], tokens[2])
class ShowIfCachedNode(Node):
def __init__(self, key):
self.key = key
def render(self, context):
key = resolve_variable(self.key, context)
return cache.get(key) or ''
@register.tag
def showifcached(parser, token):
"""
Show content if it exists in the cache, otherwise display nothing.
The key is entirely deterministic and not mangled or run through MD5 (cf.
{% cache %})
Usage::
{% showifcached [key] %}
"""
tokens = token.contents.split()
if len(tokens) != 2:
raise TemplateSyntaxError(u"'%r' tag requires 1 argument." % tokens[0])
return ShowIfCachedNode(tokens[1])

0
lms/lib/util/__init__.py
View File

16
lms/lib/util/cache.py
View File

@@ -1,16 +0,0 @@
"""
This module aims to give a little more fine-tuned control of caching and cache
invalidation. Import these instead of django.core.cache.
Note that 'default' is being preserved for user session caching, which we're
not migrating so as not to inconvenience users by logging them all out.
"""
from django.core import cache
# If we can't find a 'general' CACHE defined in settings.py, we simply fall back
# to returning the default cache. This will happen with dev machines.
try:
cache = cache.get_cache('general')
except Exception:
cache = cache.cache

20
lms/lib/util/memcache.py
View File

@@ -1,20 +0,0 @@
"""
This module provides a KEY_FUNCTION suitable for use with a memcache backend
so that we can cache any keys, not just ones that memcache would ordinarily accept
"""
from django.utils.encoding import smart_str
import hashlib
import urllib
def fasthash(string):
m = hashlib.new("md4")
m.update(string)
return m.hexdigest()
def safe_key(key, key_prefix, version):
safe_key = urllib.quote_plus(smart_str(key))
if len(safe_key) > 250:
safe_key = fasthash(safe_key)
return ":".join([key_prefix, str(version), safe_key])

16
lms/lib/util/middleware.py
View File

@@ -1,16 +0,0 @@
import logging
from django.conf import settings
from django.http import HttpResponseServerError
log = logging.getLogger("mitx")
class ExceptionLoggingMiddleware(object):
"""Just here to log unchecked exceptions that go all the way up the Django
stack"""
if not settings.TEMPLATE_DEBUG:
def process_exception(self, request, exception):
log.exception(exception)
return HttpResponseServerError("Server Error - Please try again later.")

3
lms/lib/util/models.py
View File

@@ -1,3 +0,0 @@
from django.db import models
# Create your models here.

16
lms/lib/util/tests.py
View File

@@ -1,16 +0,0 @@
"""
This file demonstrates writing tests using the unittest module. These will pass
when you run "manage.py test".
Replace this with more appropriate tests for your application.
"""
from django.test import TestCase
class SimpleTest(TestCase):
def test_basic_addition(self):
"""
Tests that 1 + 1 always equals 2.
"""
self.assertEqual(1 + 1, 2)

88
lms/lib/util/views.py
View File

@@ -1,88 +0,0 @@
import datetime
import json
import sys
from django.conf import settings
from django.contrib.auth.models import User
from django.core.context_processors import csrf
from django.core.mail import send_mail
from django.http import Http404
from django.http import HttpResponse
from django.shortcuts import redirect
from mitxmako.shortcuts import render_to_response, render_to_string
import capa.calc
import track.views
def calculate(request):
''' Calculator in footer of every page. '''
equation = request.GET['equation']
try:
result = capa.calc.evaluator({}, {}, equation)
except:
event = {'error':map(str,sys.exc_info()),
'equation':equation}
track.views.server_track(request, 'error:calc', event, page='calc')
return HttpResponse(json.dumps({'result':'Invalid syntax'}))
return HttpResponse(json.dumps({'result':str(result)}))
def send_feedback(request):
''' Feeback mechanism in footer of every page. '''
try:
username = request.user.username
email = request.user.email
except:
username = "anonymous"
email = "anonymous"
try:
browser = request.META['HTTP_USER_AGENT']
except:
browser = "Unknown"
feedback = render_to_string("feedback_email.txt",
{"subject":request.POST['subject'],
"url": request.POST['url'],
"time": datetime.datetime.now().isoformat(),
"feedback": request.POST['message'],
"email":email,
"browser":browser,
"user":username})
send_mail("MITx Feedback / " +request.POST['subject'],
feedback,
settings.DEFAULT_FROM_EMAIL,
[ settings.DEFAULT_FEEDBACK_EMAIL ],
fail_silently = False
)
return HttpResponse(json.dumps({'success':True}))
def info(request):
''' Info page (link from main header) '''
return render_to_response("info.html", {})
# From http://djangosnippets.org/snippets/1042/
def parse_accept_header(accept):
"""Parse the Accept header *accept*, returning a list with pairs of
(media_type, q_value), ordered by q values.
"""
result = []
for media_range in accept.split(","):
parts = media_range.split(";")
media_type = parts.pop(0)
media_params = []
q = 1.0
for part in parts:
(key, value) = part.lstrip().split("=", 1)
if key == "q":
q = float(value)
else:
media_params.append((key, value))
result.append((media_type, tuple(media_params), q))
result.sort(lambda x, y: -cmp(x[2], y[2]))
return result
def accepts(request, media_type):
"""Return whether this request has an Accept header that matches type"""
accept = parse_accept_header(request.META.get("HTTP_ACCEPT", ""))
return media_type in [t for (t, p, q) in accept]