Added base Celery settings

2013-03-25 17:18:02 -04:00
parent 95496b5c50
commit c05efbc31e
12 changed files with 319 additions and 10 deletions
--- a/cms/envs/aws.py
+++ b/cms/envs/aws.py
@@ -28,6 +28,47 @@ EMAIL_BACKEND = 'django_ses.SESBackend'
 SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
 DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage'

+###################################### CELERY  ################################
+
+# Don't use a connection pool, since connections are dropped by ELB.
+BROKER_POOL_LIMIT = 0
+BROKER_CONNECTION_TIMEOUT = 1
+
+# For the Result Store, use the django cache named 'celery'
+CELERY_RESULT_BACKEND = 'cache'
+CELERY_CACHE_BACKEND = 'celery'
+
+# When the broker is behind an ELB, use a heartbeat to refresh the
+# connection and to detect if it has been dropped.
+BROKER_HEARTBEAT = 10.0
+BROKER_HEARTBEAT_CHECKRATE = 2
+
+# Each worker should only fetch one message at a time
+CELERYD_PREFETCH_MULTIPLIER = 1
+
+# Skip djcelery migrations, since we don't use the database as the broker
+SOUTH_MIGRATION_MODULES = {
+    'djcelery': 'ignore',
+}
+
+# Rename the exchange and queues for each variant
+
+QUEUE_VARIANT = CONFIG_PREFIX.lower()
+
+CELERY_DEFAULT_EXCHANGE = 'edx.{0}core'.format(QUEUE_VARIANT)
+
+HIGH_PRIORITY_QUEUE = 'edx.{0}core.high'.format(QUEUE_VARIANT)
+DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default'.format(QUEUE_VARIANT)
+LOW_PRIORITY_QUEUE = 'edx.{0}core.low'.format(QUEUE_VARIANT)
+
+CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE
+
+CELERY_QUEUES = {
+    HIGH_PRIORITY_QUEUE: {},
+    LOW_PRIORITY_QUEUE: {},
+    DEFAULT_PRIORITY_QUEUE: {}
+}
+
 ############# NON-SECURE ENV CONFIG ##############################
 # Things like server locations, ports, etc.
 with open(ENV_ROOT / CONFIG_PREFIX + "env.json") as env_file:
@@ -78,3 +119,14 @@ CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE']

 # Datadog for events!
 DATADOG_API = AUTH_TOKENS.get("DATADOG_API")
+
+# Celery Broker
+CELERY_BROKER_TRANSPORT = ENV_TOKENS.get("CELERY_BROKER_TRANSPORT", "")
+CELERY_BROKER_HOSTNAME = ENV_TOKENS.get("CELERY_BROKER_HOSTNAME", "")
+CELERY_BROKER_USER = AUTH_TOKENS.get("CELERY_BROKER_USER", "")
+CELERY_BROKER_PASSWORD = AUTH_TOKENS.get("CELERY_BROKER_PASSWORD", "")
+
+BROKER_URL = "{0}://{1}:{2}@{3}".format(CELERY_BROKER_TRANSPORT,
+                                        CELERY_BROKER_USER,
+                                        CELERY_BROKER_PASSWORD,
+                                        CELERY_BROKER_HOSTNAME)
--- a/cms/envs/common.py
+++ b/cms/envs/common.py
@@ -240,6 +240,51 @@ STATICFILES_IGNORE_PATTERNS = (

 PIPELINE_YUI_BINARY = 'yui-compressor'

+################################# CELERY ######################################
+
+# Message configuration
+
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'
+
+CELERY_MESSAGE_COMPRESSION = 'gzip'
+
+# Results configuration
+
+CELERY_IGNORE_RESULT = False
+CELERY_STORE_ERRORS_EVEN_IF_IGNORED = True
+
+# Events configuration
+
+CELERY_TRACK_STARTED = True
+
+CELERY_SEND_EVENTS = True
+CELERY_SEND_TASK_SENT_EVENT = True
+
+# Exchange configuration
+
+CELERY_DEFAULT_EXCHANGE = 'edx.core'
+CELERY_DEFAULT_EXCHANGE_TYPE = 'direct'
+
+# Queues configuration
+
+HIGH_PRIORITY_QUEUE = 'edx.core.high'
+DEFAULT_PRIORITY_QUEUE = 'edx.core.default'
+LOW_PRIORITY_QUEUE = 'edx.core.low'
+
+CELERY_QUEUE_HA_POLICY = 'all'
+
+CELERY_CREATE_MISSING_QUEUES = True
+
+CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE
+CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE
+
+CELERY_QUEUES = {
+    HIGH_PRIORITY_QUEUE: {},
+    LOW_PRIORITY_QUEUE: {},
+    DEFAULT_PRIORITY_QUEUE: {}
+}
+
 ############################ APPS #####################################

 INSTALLED_APPS = (
@@ -249,8 +294,12 @@ INSTALLED_APPS = (
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
+    'djcelery',
    'south',

+    # Monitor the status of services
+    'service_status',
+
    # For CMS
    'contentstore',
    'auth',
--- a/cms/envs/dev.py
+++ b/cms/envs/dev.py
@@ -116,6 +116,11 @@ SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'

 PIPELINE_SASS_ARGUMENTS = '--debug-info --require {proj_dir}/static/sass/bourbon/lib/bourbon.rb'.format(proj_dir=PROJECT_ROOT)

+################################# CELERY ######################################
+
+# By default don't use a worker, execute tasks as if they were local functions
+CELERY_ALWAYS_EAGER = True
+
 ################################ DEBUG TOOLBAR #################################
 INSTALLED_APPS += ('debug_toolbar', 'debug_toolbar_mongo')
 MIDDLEWARE_CLASSES += ('django_comment_client.utils.QueryCountDebugMiddleware',
--- a/cms/envs/dev_with_worker.py
+++ b/cms/envs/dev_with_worker.py
@@ -0,0 +1,35 @@
+"""
+This config file follows the dev enviroment, but adds the
+requirement of a celery worker running in the background to process
+celery tasks.
+
+The worker can be executed using:
+
+django_admin.py celery worker
+"""
+
+from dev import *
+
+################################# CELERY ######################################
+
+# Requires a separate celery worker
+
+CELERY_ALWAYS_EAGER = False
+
+# Use django db as the broker and result store
+
+BROKER_URL = 'django://'
+INSTALLED_APPS += ('djcelery.transport', )
+CELERY_RESULT_BACKEND = 'database'
+DJKOMBU_POLLING_INTERVAL = 1.0
+
+# Disable transaction management because we are using a worker. Views
+# that request a task and wait for the result will deadlock otherwise.
+
+MIDDLEWARE_CLASSES = tuple(
+    c for c in MIDDLEWARE_CLASSES
+    if c != 'django.middleware.transaction.TransactionMiddleware')
+
+# Note: other alternatives for disabling transactions don't work in 1.4
+# https://code.djangoproject.com/ticket/2304
+# https://code.djangoproject.com/ticket/16039
--- a/cms/envs/test.py
+++ b/cms/envs/test.py
@@ -108,6 +108,12 @@ CACHES = {
    }
 }

+################################# CELERY ######################################
+
+CELERY_ALWAYS_EAGER = True
+CELERY_RESULT_BACKEND = 'cache'
+BROKER_TRANSPORT = 'memory'
+
 ################### Make tests faster
 #http://slacy.com/blog/2012/04/make-your-tests-faster-in-django-1-4/
 PASSWORD_HASHERS = (
@@ -120,4 +126,3 @@ SEGMENT_IO_KEY = '***REMOVED***'

 # disable NPS survey in test mode
 MITX_FEATURES['STUDIO_NPS_SURVEY'] = False
-
--- a/lms/envs/aws.py
+++ b/lms/envs/aws.py
@@ -26,7 +26,8 @@ if SERVICE_VARIANT:
    CONFIG_PREFIX = SERVICE_VARIANT + "."


-################### ALWAYS THE SAME ################################
+################################ ALWAYS THE SAME ##############################
+
 DEBUG = False
 TEMPLATE_DEBUG = False

@@ -45,7 +46,48 @@ MITX_FEATURES['ENABLE_DISCUSSION_SERVICE'] = True
 # for other warnings.
 SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')

-################# NON-SECURE ENV CONFIG ##############################
+###################################### CELERY  ################################
+
+# Don't use a connection pool, since connections are dropped by ELB.
+BROKER_POOL_LIMIT = 0
+BROKER_CONNECTION_TIMEOUT = 1
+
+# For the Result Store, use the django cache named 'celery'
+CELERY_RESULT_BACKEND = 'cache'
+CELERY_CACHE_BACKEND = 'celery'
+
+# When the broker is behind an ELB, use a heartbeat to refresh the
+# connection and to detect if it has been dropped.
+BROKER_HEARTBEAT = 10.0
+BROKER_HEARTBEAT_CHECKRATE = 2
+
+# Each worker should only fetch one message at a time
+CELERYD_PREFETCH_MULTIPLIER = 1
+
+# Skip djcelery migrations, since we don't use the database as the broker
+SOUTH_MIGRATION_MODULES = {
+    'djcelery': 'ignore',
+}
+
+# Rename the exchange and queues for each variant
+
+QUEUE_VARIANT = CONFIG_PREFIX.lower()
+
+CELERY_DEFAULT_EXCHANGE = 'edx.{0}core'.format(QUEUE_VARIANT)
+
+HIGH_PRIORITY_QUEUE = 'edx.{0}core.high'.format(QUEUE_VARIANT)
+DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default'.format(QUEUE_VARIANT)
+LOW_PRIORITY_QUEUE = 'edx.{0}core.low'.format(QUEUE_VARIANT)
+
+CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE
+
+CELERY_QUEUES = {
+    HIGH_PRIORITY_QUEUE: {},
+    LOW_PRIORITY_QUEUE: {},
+    DEFAULT_PRIORITY_QUEUE: {}
+}
+
+########################## NON-SECURE ENV CONFIG ##############################
 # Things like server locations, ports, etc.

 with open(ENV_ROOT / CONFIG_PREFIX + "env.json") as env_file:
@@ -104,6 +146,7 @@ COURSES_WITH_UNSAFE_CODE = ENV_TOKENS.get("COURSES_WITH_UNSAFE_CODE", [])

 ############################## SECURE AUTH ITEMS ###############
 # Secret things: passwords, access keys, etc.
+
 with open(ENV_ROOT / CONFIG_PREFIX + "auth.json") as auth_file:
    AUTH_TOKENS = json.load(auth_file)

@@ -122,7 +165,8 @@ XQUEUE_INTERFACE = AUTH_TOKENS['XQUEUE_INTERFACE']
 MODULESTORE = AUTH_TOKENS.get('MODULESTORE', MODULESTORE)
 CONTENTSTORE = AUTH_TOKENS.get('CONTENTSTORE', CONTENTSTORE)

-OPEN_ENDED_GRADING_INTERFACE = AUTH_TOKENS.get('OPEN_ENDED_GRADING_INTERFACE', OPEN_ENDED_GRADING_INTERFACE)
+OPEN_ENDED_GRADING_INTERFACE = AUTH_TOKENS.get('OPEN_ENDED_GRADING_INTERFACE',
+                                               OPEN_ENDED_GRADING_INTERFACE)

 PEARSON_TEST_USER = "pearsontest"
 PEARSON_TEST_PASSWORD = AUTH_TOKENS.get("PEARSON_TEST_PASSWORD")
@@ -137,5 +181,17 @@ DATADOG_API = AUTH_TOKENS.get("DATADOG_API")
 ANALYTICS_SERVER_URL = ENV_TOKENS.get("ANALYTICS_SERVER_URL")
 ANALYTICS_API_KEY = AUTH_TOKENS.get("ANALYTICS_API_KEY", "")

+# Zendesk
 ZENDESK_USER = AUTH_TOKENS.get("ZENDESK_USER")
 ZENDESK_API_KEY = AUTH_TOKENS.get("ZENDESK_API_KEY")
+
+# Celery Broker
+CELERY_BROKER_TRANSPORT = ENV_TOKENS.get("CELERY_BROKER_TRANSPORT", "")
+CELERY_BROKER_HOSTNAME = ENV_TOKENS.get("CELERY_BROKER_HOSTNAME", "")
+CELERY_BROKER_USER = AUTH_TOKENS.get("CELERY_BROKER_USER", "")
+CELERY_BROKER_PASSWORD = AUTH_TOKENS.get("CELERY_BROKER_PASSWORD", "")
+
+BROKER_URL = "{0}://{1}:{2}@{3}".format(CELERY_BROKER_TRANSPORT,
+                                        CELERY_BROKER_USER,
+                                        CELERY_BROKER_PASSWORD,
+                                        CELERY_BROKER_HOSTNAME)
--- a/lms/envs/common.py
+++ b/lms/envs/common.py
@@ -582,7 +582,52 @@ PIPELINE_YUI_BINARY = 'yui-compressor'
 # Setting that will only affect the MITx version of django-pipeline until our changes are merged upstream
 PIPELINE_COMPILE_INPLACE = True

-################################### APPS #######################################
+################################# CELERY ######################################
+
+# Message configuration
+
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'
+
+CELERY_MESSAGE_COMPRESSION = 'gzip'
+
+# Results configuration
+
+CELERY_IGNORE_RESULT = False
+CELERY_STORE_ERRORS_EVEN_IF_IGNORED = True
+
+# Events configuration
+
+CELERY_TRACK_STARTED = True
+
+CELERY_SEND_EVENTS = True
+CELERY_SEND_TASK_SENT_EVENT = True
+
+# Exchange configuration
+
+CELERY_DEFAULT_EXCHANGE = 'edx.core'
+CELERY_DEFAULT_EXCHANGE_TYPE = 'direct'
+
+# Queues configuration
+
+HIGH_PRIORITY_QUEUE = 'edx.core.high'
+DEFAULT_PRIORITY_QUEUE = 'edx.core.default'
+LOW_PRIORITY_QUEUE = 'edx.core.low'
+
+CELERY_QUEUE_HA_POLICY = 'all'
+
+CELERY_CREATE_MISSING_QUEUES = True
+
+CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE
+CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE
+
+CELERY_QUEUES = {
+    HIGH_PRIORITY_QUEUE: {},
+    LOW_PRIORITY_QUEUE: {},
+    DEFAULT_PRIORITY_QUEUE: {}
+}
+
+################################### APPS ######################################
 INSTALLED_APPS = (
    # Standard ones that are always installed...
    'django.contrib.auth',
@@ -591,8 +636,12 @@ INSTALLED_APPS = (
    'django.contrib.messages',
    'django.contrib.sessions',
    'django.contrib.sites',
+    'djcelery',
    'south',

+    # Monitor the status of services
+    'service_status',
+
    # For asset pipelining
    'pipeline',
    'staticfiles',
@@ -639,4 +688,3 @@ INSTALLED_APPS = (
    # Student notes
    'notes',
 )
-
--- a/lms/envs/dev.py
+++ b/lms/envs/dev.py
@@ -163,6 +163,7 @@ INSTALLED_APPS += ('lms_migration',)
 LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1']

 ################################ OpenID Auth #################################
+
 MITX_FEATURES['AUTH_USE_OPENID'] = True
 MITX_FEATURES['AUTH_USE_OPENID_PROVIDER'] = True
 MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True
@@ -177,11 +178,17 @@ OPENID_USE_AS_ADMIN_LOGIN = False

 OPENID_PROVIDER_TRUSTED_ROOTS = ['*']

-################################ MIT Certificates SSL Auth #################################
+######################## MIT Certificates SSL Auth ############################

 MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True

-################################ DEBUG TOOLBAR #################################
+################################# CELERY ######################################
+
+# By default don't use a worker, execute tasks as if they were local functions
+CELERY_ALWAYS_EAGER = True
+
+################################ DEBUG TOOLBAR ################################
+
 INSTALLED_APPS += ('debug_toolbar',)
 MIDDLEWARE_CLASSES += ('django_comment_client.utils.QueryCountDebugMiddleware',
                       'debug_toolbar.middleware.DebugToolbarMiddleware',)
@@ -207,7 +214,9 @@ DEBUG_TOOLBAR_PANELS = (
 DEBUG_TOOLBAR_CONFIG = {
    'INTERCEPT_REDIRECTS': False
 }
+
 #################### FILE UPLOADS (for discussion forums) #####################
+
 DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
 MEDIA_ROOT = ENV_ROOT / "uploads"
 MEDIA_URL = "/static/uploads/"
--- a/lms/envs/dev_with_worker.py
+++ b/lms/envs/dev_with_worker.py
@@ -0,0 +1,35 @@
+"""
+This config file follows the dev enviroment, but adds the
+requirement of a celery worker running in the background to process
+celery tasks.
+
+The worker can be executed using:
+
+django_admin.py celery worker
+"""
+
+from dev import *
+
+################################# CELERY ######################################
+
+# Requires a separate celery worker
+
+CELERY_ALWAYS_EAGER = False
+
+# Use django db as the broker and result store
+
+BROKER_URL = 'django://'
+INSTALLED_APPS += ('djcelery.transport', )
+CELERY_RESULT_BACKEND = 'database'
+DJKOMBU_POLLING_INTERVAL = 1.0
+
+# Disable transaction management because we are using a worker. Views
+# that request a task and wait for the result will deadlock otherwise.
+
+MIDDLEWARE_CLASSES = tuple(
+    c for c in MIDDLEWARE_CLASSES
+    if c != 'django.middleware.transaction.TransactionMiddleware')
+
+# Note: other alternatives for disabling transactions don't work in 1.4
+# https://code.djangoproject.com/ticket/2304
+# https://code.djangoproject.com/ticket/16039
--- a/lms/envs/test.py
+++ b/lms/envs/test.py
@@ -124,7 +124,7 @@ CACHES = {
 # Dummy secret key for dev
 SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'

-################################## OPENID ######################################
+################################## OPENID #####################################
 MITX_FEATURES['AUTH_USE_OPENID'] = True
 MITX_FEATURES['AUTH_USE_OPENID_PROVIDER'] = True

@@ -136,6 +136,12 @@ OPENID_PROVIDER_TRUSTED_ROOTS = ['*']
 INSTALLED_APPS += ('external_auth',)
 INSTALLED_APPS += ('django_openid_auth',)

+################################# CELERY ######################################
+
+CELERY_ALWAYS_EAGER = True
+CELERY_RESULT_BACKEND = 'cache'
+BROKER_TRANSPORT = 'memory'
+
 ############################ STATIC FILES #############################
 DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
 MEDIA_ROOT = TEST_ROOT / "uploads"
--- a/rakefiles/django.rake
+++ b/rakefiles/django.rake
@@ -25,6 +25,14 @@ end
        sh(django_admin(system, args.env, 'runserver', args.options))
    end

+    desc "Start #{system} Celery worker"
+    task "#{system}_worker", [:options] => [:predjango] do |t, args|
+      args.with_defaults(:options => default_options[system])
+      django_admin = ENV['DJANGO_ADMIN_PATH'] || select_executable('django-admin.py', 'django-admin')
+      command = 'celery worker'
+      sh("#{django_admin} #{command} --loglevel=INFO --settings=#{system}.envs.dev_with_worker --pythonpath=. #{args.join(' ')}")
+    end
+
    # Per environment tasks
    environments(system).each do |env|
        desc "Attempt to import the settings file #{system}.envs.#{env} and report any errors"
--- a/requirements/edx/base.txt
+++ b/requirements/edx/base.txt
@@ -3,8 +3,9 @@
 beautifulsoup4==4.1.3
 beautifulsoup==3.2.1
 boto==2.6.0
+celery==3.0.19
 distribute==0.6.28
-django-celery==3.0.11
+django-celery==3.0.17
 django-countries==1.5
 django-followit==0.0.3
 django-keyedcache==1.4-6