From c05efbc31ee598a734d2a2803a93d4af22e36a11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Andr=C3=A9s=20Rocha?= Date: Mon, 25 Mar 2013 17:18:02 -0400 Subject: [PATCH] Added base Celery settings --- cms/envs/aws.py | 52 +++++++++++++++++++++++++++++++ cms/envs/common.py | 49 +++++++++++++++++++++++++++++ cms/envs/dev.py | 5 +++ cms/envs/dev_with_worker.py | 35 +++++++++++++++++++++ cms/envs/test.py | 7 ++++- lms/envs/aws.py | 62 +++++++++++++++++++++++++++++++++++-- lms/envs/common.py | 52 +++++++++++++++++++++++++++++-- lms/envs/dev.py | 13 ++++++-- lms/envs/dev_with_worker.py | 35 +++++++++++++++++++++ lms/envs/test.py | 8 ++++- rakefiles/django.rake | 8 +++++ requirements/edx/base.txt | 3 +- 12 files changed, 319 insertions(+), 10 deletions(-) create mode 100644 cms/envs/dev_with_worker.py create mode 100644 lms/envs/dev_with_worker.py diff --git a/cms/envs/aws.py b/cms/envs/aws.py index 05c57d8263..082f63b812 100644 --- a/cms/envs/aws.py +++ b/cms/envs/aws.py @@ -28,6 +28,47 @@ EMAIL_BACKEND = 'django_ses.SESBackend' SESSION_ENGINE = 'django.contrib.sessions.backends.cache' DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage' +###################################### CELERY ################################ + +# Don't use a connection pool, since connections are dropped by ELB. +BROKER_POOL_LIMIT = 0 +BROKER_CONNECTION_TIMEOUT = 1 + +# For the Result Store, use the django cache named 'celery' +CELERY_RESULT_BACKEND = 'cache' +CELERY_CACHE_BACKEND = 'celery' + +# When the broker is behind an ELB, use a heartbeat to refresh the +# connection and to detect if it has been dropped. +BROKER_HEARTBEAT = 10.0 +BROKER_HEARTBEAT_CHECKRATE = 2 + +# Each worker should only fetch one message at a time +CELERYD_PREFETCH_MULTIPLIER = 1 + +# Skip djcelery migrations, since we don't use the database as the broker +SOUTH_MIGRATION_MODULES = { + 'djcelery': 'ignore', +} + +# Rename the exchange and queues for each variant + +QUEUE_VARIANT = CONFIG_PREFIX.lower() + +CELERY_DEFAULT_EXCHANGE = 'edx.{0}core'.format(QUEUE_VARIANT) + +HIGH_PRIORITY_QUEUE = 'edx.{0}core.high'.format(QUEUE_VARIANT) +DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default'.format(QUEUE_VARIANT) +LOW_PRIORITY_QUEUE = 'edx.{0}core.low'.format(QUEUE_VARIANT) + +CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE + +CELERY_QUEUES = { + HIGH_PRIORITY_QUEUE: {}, + LOW_PRIORITY_QUEUE: {}, + DEFAULT_PRIORITY_QUEUE: {} +} + ############# NON-SECURE ENV CONFIG ############################## # Things like server locations, ports, etc. with open(ENV_ROOT / CONFIG_PREFIX + "env.json") as env_file: @@ -78,3 +119,14 @@ CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE'] # Datadog for events! DATADOG_API = AUTH_TOKENS.get("DATADOG_API") + +# Celery Broker +CELERY_BROKER_TRANSPORT = ENV_TOKENS.get("CELERY_BROKER_TRANSPORT", "") +CELERY_BROKER_HOSTNAME = ENV_TOKENS.get("CELERY_BROKER_HOSTNAME", "") +CELERY_BROKER_USER = AUTH_TOKENS.get("CELERY_BROKER_USER", "") +CELERY_BROKER_PASSWORD = AUTH_TOKENS.get("CELERY_BROKER_PASSWORD", "") + +BROKER_URL = "{0}://{1}:{2}@{3}".format(CELERY_BROKER_TRANSPORT, + CELERY_BROKER_USER, + CELERY_BROKER_PASSWORD, + CELERY_BROKER_HOSTNAME) diff --git a/cms/envs/common.py b/cms/envs/common.py index a53830082b..fb61db4d73 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -240,6 +240,51 @@ STATICFILES_IGNORE_PATTERNS = ( PIPELINE_YUI_BINARY = 'yui-compressor' +################################# CELERY ###################################### + +# Message configuration + +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' + +CELERY_MESSAGE_COMPRESSION = 'gzip' + +# Results configuration + +CELERY_IGNORE_RESULT = False +CELERY_STORE_ERRORS_EVEN_IF_IGNORED = True + +# Events configuration + +CELERY_TRACK_STARTED = True + +CELERY_SEND_EVENTS = True +CELERY_SEND_TASK_SENT_EVENT = True + +# Exchange configuration + +CELERY_DEFAULT_EXCHANGE = 'edx.core' +CELERY_DEFAULT_EXCHANGE_TYPE = 'direct' + +# Queues configuration + +HIGH_PRIORITY_QUEUE = 'edx.core.high' +DEFAULT_PRIORITY_QUEUE = 'edx.core.default' +LOW_PRIORITY_QUEUE = 'edx.core.low' + +CELERY_QUEUE_HA_POLICY = 'all' + +CELERY_CREATE_MISSING_QUEUES = True + +CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE +CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE + +CELERY_QUEUES = { + HIGH_PRIORITY_QUEUE: {}, + LOW_PRIORITY_QUEUE: {}, + DEFAULT_PRIORITY_QUEUE: {} +} + ############################ APPS ##################################### INSTALLED_APPS = ( @@ -249,8 +294,12 @@ INSTALLED_APPS = ( 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.messages', + 'djcelery', 'south', + # Monitor the status of services + 'service_status', + # For CMS 'contentstore', 'auth', diff --git a/cms/envs/dev.py b/cms/envs/dev.py index dbf9c5574c..a05ad5731b 100644 --- a/cms/envs/dev.py +++ b/cms/envs/dev.py @@ -116,6 +116,11 @@ SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' PIPELINE_SASS_ARGUMENTS = '--debug-info --require {proj_dir}/static/sass/bourbon/lib/bourbon.rb'.format(proj_dir=PROJECT_ROOT) +################################# CELERY ###################################### + +# By default don't use a worker, execute tasks as if they were local functions +CELERY_ALWAYS_EAGER = True + ################################ DEBUG TOOLBAR ################################# INSTALLED_APPS += ('debug_toolbar', 'debug_toolbar_mongo') MIDDLEWARE_CLASSES += ('django_comment_client.utils.QueryCountDebugMiddleware', diff --git a/cms/envs/dev_with_worker.py b/cms/envs/dev_with_worker.py new file mode 100644 index 0000000000..c5fc256ac9 --- /dev/null +++ b/cms/envs/dev_with_worker.py @@ -0,0 +1,35 @@ +""" +This config file follows the dev enviroment, but adds the +requirement of a celery worker running in the background to process +celery tasks. + +The worker can be executed using: + +django_admin.py celery worker +""" + +from dev import * + +################################# CELERY ###################################### + +# Requires a separate celery worker + +CELERY_ALWAYS_EAGER = False + +# Use django db as the broker and result store + +BROKER_URL = 'django://' +INSTALLED_APPS += ('djcelery.transport', ) +CELERY_RESULT_BACKEND = 'database' +DJKOMBU_POLLING_INTERVAL = 1.0 + +# Disable transaction management because we are using a worker. Views +# that request a task and wait for the result will deadlock otherwise. + +MIDDLEWARE_CLASSES = tuple( + c for c in MIDDLEWARE_CLASSES + if c != 'django.middleware.transaction.TransactionMiddleware') + +# Note: other alternatives for disabling transactions don't work in 1.4 +# https://code.djangoproject.com/ticket/2304 +# https://code.djangoproject.com/ticket/16039 diff --git a/cms/envs/test.py b/cms/envs/test.py index 06ea5309ed..4362aaaac0 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -108,6 +108,12 @@ CACHES = { } } +################################# CELERY ###################################### + +CELERY_ALWAYS_EAGER = True +CELERY_RESULT_BACKEND = 'cache' +BROKER_TRANSPORT = 'memory' + ################### Make tests faster #http://slacy.com/blog/2012/04/make-your-tests-faster-in-django-1-4/ PASSWORD_HASHERS = ( @@ -120,4 +126,3 @@ SEGMENT_IO_KEY = '***REMOVED***' # disable NPS survey in test mode MITX_FEATURES['STUDIO_NPS_SURVEY'] = False - diff --git a/lms/envs/aws.py b/lms/envs/aws.py index 74540b7dec..d60215bf7b 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -26,7 +26,8 @@ if SERVICE_VARIANT: CONFIG_PREFIX = SERVICE_VARIANT + "." -################### ALWAYS THE SAME ################################ +################################ ALWAYS THE SAME ############################## + DEBUG = False TEMPLATE_DEBUG = False @@ -45,7 +46,48 @@ MITX_FEATURES['ENABLE_DISCUSSION_SERVICE'] = True # for other warnings. SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') -################# NON-SECURE ENV CONFIG ############################## +###################################### CELERY ################################ + +# Don't use a connection pool, since connections are dropped by ELB. +BROKER_POOL_LIMIT = 0 +BROKER_CONNECTION_TIMEOUT = 1 + +# For the Result Store, use the django cache named 'celery' +CELERY_RESULT_BACKEND = 'cache' +CELERY_CACHE_BACKEND = 'celery' + +# When the broker is behind an ELB, use a heartbeat to refresh the +# connection and to detect if it has been dropped. +BROKER_HEARTBEAT = 10.0 +BROKER_HEARTBEAT_CHECKRATE = 2 + +# Each worker should only fetch one message at a time +CELERYD_PREFETCH_MULTIPLIER = 1 + +# Skip djcelery migrations, since we don't use the database as the broker +SOUTH_MIGRATION_MODULES = { + 'djcelery': 'ignore', +} + +# Rename the exchange and queues for each variant + +QUEUE_VARIANT = CONFIG_PREFIX.lower() + +CELERY_DEFAULT_EXCHANGE = 'edx.{0}core'.format(QUEUE_VARIANT) + +HIGH_PRIORITY_QUEUE = 'edx.{0}core.high'.format(QUEUE_VARIANT) +DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default'.format(QUEUE_VARIANT) +LOW_PRIORITY_QUEUE = 'edx.{0}core.low'.format(QUEUE_VARIANT) + +CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE + +CELERY_QUEUES = { + HIGH_PRIORITY_QUEUE: {}, + LOW_PRIORITY_QUEUE: {}, + DEFAULT_PRIORITY_QUEUE: {} +} + +########################## NON-SECURE ENV CONFIG ############################## # Things like server locations, ports, etc. with open(ENV_ROOT / CONFIG_PREFIX + "env.json") as env_file: @@ -104,6 +146,7 @@ COURSES_WITH_UNSAFE_CODE = ENV_TOKENS.get("COURSES_WITH_UNSAFE_CODE", []) ############################## SECURE AUTH ITEMS ############### # Secret things: passwords, access keys, etc. + with open(ENV_ROOT / CONFIG_PREFIX + "auth.json") as auth_file: AUTH_TOKENS = json.load(auth_file) @@ -122,7 +165,8 @@ XQUEUE_INTERFACE = AUTH_TOKENS['XQUEUE_INTERFACE'] MODULESTORE = AUTH_TOKENS.get('MODULESTORE', MODULESTORE) CONTENTSTORE = AUTH_TOKENS.get('CONTENTSTORE', CONTENTSTORE) -OPEN_ENDED_GRADING_INTERFACE = AUTH_TOKENS.get('OPEN_ENDED_GRADING_INTERFACE', OPEN_ENDED_GRADING_INTERFACE) +OPEN_ENDED_GRADING_INTERFACE = AUTH_TOKENS.get('OPEN_ENDED_GRADING_INTERFACE', + OPEN_ENDED_GRADING_INTERFACE) PEARSON_TEST_USER = "pearsontest" PEARSON_TEST_PASSWORD = AUTH_TOKENS.get("PEARSON_TEST_PASSWORD") @@ -137,5 +181,17 @@ DATADOG_API = AUTH_TOKENS.get("DATADOG_API") ANALYTICS_SERVER_URL = ENV_TOKENS.get("ANALYTICS_SERVER_URL") ANALYTICS_API_KEY = AUTH_TOKENS.get("ANALYTICS_API_KEY", "") +# Zendesk ZENDESK_USER = AUTH_TOKENS.get("ZENDESK_USER") ZENDESK_API_KEY = AUTH_TOKENS.get("ZENDESK_API_KEY") + +# Celery Broker +CELERY_BROKER_TRANSPORT = ENV_TOKENS.get("CELERY_BROKER_TRANSPORT", "") +CELERY_BROKER_HOSTNAME = ENV_TOKENS.get("CELERY_BROKER_HOSTNAME", "") +CELERY_BROKER_USER = AUTH_TOKENS.get("CELERY_BROKER_USER", "") +CELERY_BROKER_PASSWORD = AUTH_TOKENS.get("CELERY_BROKER_PASSWORD", "") + +BROKER_URL = "{0}://{1}:{2}@{3}".format(CELERY_BROKER_TRANSPORT, + CELERY_BROKER_USER, + CELERY_BROKER_PASSWORD, + CELERY_BROKER_HOSTNAME) diff --git a/lms/envs/common.py b/lms/envs/common.py index 4cd7dd9843..64b07ef855 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -582,7 +582,52 @@ PIPELINE_YUI_BINARY = 'yui-compressor' # Setting that will only affect the MITx version of django-pipeline until our changes are merged upstream PIPELINE_COMPILE_INPLACE = True -################################### APPS ####################################### +################################# CELERY ###################################### + +# Message configuration + +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' + +CELERY_MESSAGE_COMPRESSION = 'gzip' + +# Results configuration + +CELERY_IGNORE_RESULT = False +CELERY_STORE_ERRORS_EVEN_IF_IGNORED = True + +# Events configuration + +CELERY_TRACK_STARTED = True + +CELERY_SEND_EVENTS = True +CELERY_SEND_TASK_SENT_EVENT = True + +# Exchange configuration + +CELERY_DEFAULT_EXCHANGE = 'edx.core' +CELERY_DEFAULT_EXCHANGE_TYPE = 'direct' + +# Queues configuration + +HIGH_PRIORITY_QUEUE = 'edx.core.high' +DEFAULT_PRIORITY_QUEUE = 'edx.core.default' +LOW_PRIORITY_QUEUE = 'edx.core.low' + +CELERY_QUEUE_HA_POLICY = 'all' + +CELERY_CREATE_MISSING_QUEUES = True + +CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE +CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE + +CELERY_QUEUES = { + HIGH_PRIORITY_QUEUE: {}, + LOW_PRIORITY_QUEUE: {}, + DEFAULT_PRIORITY_QUEUE: {} +} + +################################### APPS ###################################### INSTALLED_APPS = ( # Standard ones that are always installed... 'django.contrib.auth', @@ -591,8 +636,12 @@ INSTALLED_APPS = ( 'django.contrib.messages', 'django.contrib.sessions', 'django.contrib.sites', + 'djcelery', 'south', + # Monitor the status of services + 'service_status', + # For asset pipelining 'pipeline', 'staticfiles', @@ -639,4 +688,3 @@ INSTALLED_APPS = ( # Student notes 'notes', ) - diff --git a/lms/envs/dev.py b/lms/envs/dev.py index dbc06efe8c..28c9a57a16 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -163,6 +163,7 @@ INSTALLED_APPS += ('lms_migration',) LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1'] ################################ OpenID Auth ################################# + MITX_FEATURES['AUTH_USE_OPENID'] = True MITX_FEATURES['AUTH_USE_OPENID_PROVIDER'] = True MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True @@ -177,11 +178,17 @@ OPENID_USE_AS_ADMIN_LOGIN = False OPENID_PROVIDER_TRUSTED_ROOTS = ['*'] -################################ MIT Certificates SSL Auth ################################# +######################## MIT Certificates SSL Auth ############################ MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True -################################ DEBUG TOOLBAR ################################# +################################# CELERY ###################################### + +# By default don't use a worker, execute tasks as if they were local functions +CELERY_ALWAYS_EAGER = True + +################################ DEBUG TOOLBAR ################################ + INSTALLED_APPS += ('debug_toolbar',) MIDDLEWARE_CLASSES += ('django_comment_client.utils.QueryCountDebugMiddleware', 'debug_toolbar.middleware.DebugToolbarMiddleware',) @@ -207,7 +214,9 @@ DEBUG_TOOLBAR_PANELS = ( DEBUG_TOOLBAR_CONFIG = { 'INTERCEPT_REDIRECTS': False } + #################### FILE UPLOADS (for discussion forums) ##################### + DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' MEDIA_ROOT = ENV_ROOT / "uploads" MEDIA_URL = "/static/uploads/" diff --git a/lms/envs/dev_with_worker.py b/lms/envs/dev_with_worker.py new file mode 100644 index 0000000000..c5fc256ac9 --- /dev/null +++ b/lms/envs/dev_with_worker.py @@ -0,0 +1,35 @@ +""" +This config file follows the dev enviroment, but adds the +requirement of a celery worker running in the background to process +celery tasks. + +The worker can be executed using: + +django_admin.py celery worker +""" + +from dev import * + +################################# CELERY ###################################### + +# Requires a separate celery worker + +CELERY_ALWAYS_EAGER = False + +# Use django db as the broker and result store + +BROKER_URL = 'django://' +INSTALLED_APPS += ('djcelery.transport', ) +CELERY_RESULT_BACKEND = 'database' +DJKOMBU_POLLING_INTERVAL = 1.0 + +# Disable transaction management because we are using a worker. Views +# that request a task and wait for the result will deadlock otherwise. + +MIDDLEWARE_CLASSES = tuple( + c for c in MIDDLEWARE_CLASSES + if c != 'django.middleware.transaction.TransactionMiddleware') + +# Note: other alternatives for disabling transactions don't work in 1.4 +# https://code.djangoproject.com/ticket/2304 +# https://code.djangoproject.com/ticket/16039 diff --git a/lms/envs/test.py b/lms/envs/test.py index 24a90e1367..9f3dea3980 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -124,7 +124,7 @@ CACHES = { # Dummy secret key for dev SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' -################################## OPENID ###################################### +################################## OPENID ##################################### MITX_FEATURES['AUTH_USE_OPENID'] = True MITX_FEATURES['AUTH_USE_OPENID_PROVIDER'] = True @@ -136,6 +136,12 @@ OPENID_PROVIDER_TRUSTED_ROOTS = ['*'] INSTALLED_APPS += ('external_auth',) INSTALLED_APPS += ('django_openid_auth',) +################################# CELERY ###################################### + +CELERY_ALWAYS_EAGER = True +CELERY_RESULT_BACKEND = 'cache' +BROKER_TRANSPORT = 'memory' + ############################ STATIC FILES ############################# DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' MEDIA_ROOT = TEST_ROOT / "uploads" diff --git a/rakefiles/django.rake b/rakefiles/django.rake index 1a021c71b8..8b42192130 100644 --- a/rakefiles/django.rake +++ b/rakefiles/django.rake @@ -25,6 +25,14 @@ end sh(django_admin(system, args.env, 'runserver', args.options)) end + desc "Start #{system} Celery worker" + task "#{system}_worker", [:options] => [:predjango] do |t, args| + args.with_defaults(:options => default_options[system]) + django_admin = ENV['DJANGO_ADMIN_PATH'] || select_executable('django-admin.py', 'django-admin') + command = 'celery worker' + sh("#{django_admin} #{command} --loglevel=INFO --settings=#{system}.envs.dev_with_worker --pythonpath=. #{args.join(' ')}") + end + # Per environment tasks environments(system).each do |env| desc "Attempt to import the settings file #{system}.envs.#{env} and report any errors" diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index f6cc250587..ef0209ee03 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -3,8 +3,9 @@ beautifulsoup4==4.1.3 beautifulsoup==3.2.1 boto==2.6.0 +celery==3.0.19 distribute==0.6.28 -django-celery==3.0.11 +django-celery==3.0.17 django-countries==1.5 django-followit==0.0.3 django-keyedcache==1.4-6