Merge branch 'master' of github.com:edx/edx-platform into feature/cdodge/autoprovision-forums-master

Conflicts:
	cms/djangoapps/contentstore/views/course.py
	lms/djangoapps/django_comment_client/base/views.py
	lms/djangoapps/django_comment_client/management/commands/show_permissions.py
	lms/djangoapps/django_comment_client/models.py
	lms/djangoapps/django_comment_client/tests/test_utils.py
	lms/envs/common.py

cms/envs/acceptance.py

@@ -8,27 +8,41 @@ from .test import *
 # otherwise the browser will not render the pages correctly
 DEBUG = True
 
-# Show the courses that are in the data directory
-COURSES_ROOT = ENV_ROOT / "data"
-DATA_DIR = COURSES_ROOT
-# MODULESTORE = {
-#     'default': {
-#         'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore',
-#         'OPTIONS': {
-#             'data_dir': DATA_DIR,
-#             'default_class': 'xmodule.hidden_module.HiddenDescriptor',
-#         }
-#     }
-# }
+# Disable warnings for acceptance tests, to make the logs readable
+import logging
+logging.disable(logging.ERROR)
 
+MODULESTORE_OPTIONS = {
+    'default_class': 'xmodule.raw_module.RawDescriptor',
+    'host': 'localhost',
+    'db': 'test_xmodule',
+    'collection': 'acceptance_modulestore',
+    'fs_root': TEST_ROOT / "data",
+    'render_template': 'mitxmako.shortcuts.render_to_string',
+}
+
+MODULESTORE = {
+    'default': {
+        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
+        'OPTIONS': MODULESTORE_OPTIONS
+    },
+    'direct': {
+        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
+        'OPTIONS': MODULESTORE_OPTIONS
+    },
+    'draft': {
+        'ENGINE': 'xmodule.modulestore.mongo.DraftMongoModuleStore',
+        'OPTIONS': MODULESTORE_OPTIONS
+    }
+}
 # Set this up so that rake lms[acceptance] and running the
 # harvest command both use the same (test) database
 # which they can flush without messing up your dev db
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': ENV_ROOT / "db" / "test_mitx.db",
-        'TEST_NAME': ENV_ROOT / "db" / "test_mitx.db",
+        'NAME': TEST_ROOT / "db" / "test_mitx.db",
+        'TEST_NAME': TEST_ROOT / "db" / "test_mitx.db",
     }
 }
 

cms/envs/aws.py

@@ -28,6 +28,48 @@ EMAIL_BACKEND = 'django_ses.SESBackend'
 SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
 DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage'
 
+###################################### CELERY  ################################
+
+# Don't use a connection pool, since connections are dropped by ELB.
+BROKER_POOL_LIMIT = 0
+BROKER_CONNECTION_TIMEOUT = 1
+
+# For the Result Store, use the django cache named 'celery'
+CELERY_RESULT_BACKEND = 'cache'
+CELERY_CACHE_BACKEND = 'celery'
+
+# When the broker is behind an ELB, use a heartbeat to refresh the
+# connection and to detect if it has been dropped.
+BROKER_HEARTBEAT = 10.0
+BROKER_HEARTBEAT_CHECKRATE = 2
+
+# Each worker should only fetch one message at a time
+CELERYD_PREFETCH_MULTIPLIER = 1
+
+# Skip djcelery migrations, since we don't use the database as the broker
+SOUTH_MIGRATION_MODULES = {
+    'djcelery': 'ignore',
+}
+
+# Rename the exchange and queues for each variant
+
+QUEUE_VARIANT = CONFIG_PREFIX.lower()
+
+CELERY_DEFAULT_EXCHANGE = 'edx.{0}core'.format(QUEUE_VARIANT)
+
+HIGH_PRIORITY_QUEUE = 'edx.{0}core.high'.format(QUEUE_VARIANT)
+DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default'.format(QUEUE_VARIANT)
+LOW_PRIORITY_QUEUE = 'edx.{0}core.low'.format(QUEUE_VARIANT)
+
+CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE
+CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE
+
+CELERY_QUEUES = {
+    HIGH_PRIORITY_QUEUE: {},
+    LOW_PRIORITY_QUEUE: {},
+    DEFAULT_PRIORITY_QUEUE: {}
+}
+
 ############# NON-SECURE ENV CONFIG ##############################
 # Things like server locations, ports, etc.
 with open(ENV_ROOT / CONFIG_PREFIX + "env.json") as env_file:
@@ -78,3 +120,14 @@ CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE']
 
 # Datadog for events!
 DATADOG_API = AUTH_TOKENS.get("DATADOG_API")
+
+# Celery Broker
+CELERY_BROKER_TRANSPORT = ENV_TOKENS.get("CELERY_BROKER_TRANSPORT", "")
+CELERY_BROKER_HOSTNAME = ENV_TOKENS.get("CELERY_BROKER_HOSTNAME", "")
+CELERY_BROKER_USER = AUTH_TOKENS.get("CELERY_BROKER_USER", "")
+CELERY_BROKER_PASSWORD = AUTH_TOKENS.get("CELERY_BROKER_PASSWORD", "")
+
+BROKER_URL = "{0}://{1}:{2}@{3}".format(CELERY_BROKER_TRANSPORT,
+                                        CELERY_BROKER_USER,
+                                        CELERY_BROKER_PASSWORD,
+                                        CELERY_BROKER_HOSTNAME)

cms/envs/common.py

@@ -34,6 +34,9 @@ MITX_FEATURES = {
     'STAFF_EMAIL': '',			# email address for staff (eg to request course creation)
     'STUDIO_NPS_SURVEY': True,
     'SEGMENT_IO': True,
+
+    # Enable URL that shows information about the status of variuous services
+    'ENABLE_SERVICE_STATUS': False,
 }
 ENABLE_JASMINE = False
 
@@ -240,6 +243,51 @@ STATICFILES_IGNORE_PATTERNS = (
 
 PIPELINE_YUI_BINARY = 'yui-compressor'
 
+################################# CELERY ######################################
+
+# Message configuration
+
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'
+
+CELERY_MESSAGE_COMPRESSION = 'gzip'
+
+# Results configuration
+
+CELERY_IGNORE_RESULT = False
+CELERY_STORE_ERRORS_EVEN_IF_IGNORED = True
+
+# Events configuration
+
+CELERY_TRACK_STARTED = True
+
+CELERY_SEND_EVENTS = True
+CELERY_SEND_TASK_SENT_EVENT = True
+
+# Exchange configuration
+
+CELERY_DEFAULT_EXCHANGE = 'edx.core'
+CELERY_DEFAULT_EXCHANGE_TYPE = 'direct'
+
+# Queues configuration
+
+HIGH_PRIORITY_QUEUE = 'edx.core.high'
+DEFAULT_PRIORITY_QUEUE = 'edx.core.default'
+LOW_PRIORITY_QUEUE = 'edx.core.low'
+
+CELERY_QUEUE_HA_POLICY = 'all'
+
+CELERY_CREATE_MISSING_QUEUES = True
+
+CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE
+CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE
+
+CELERY_QUEUES = {
+    HIGH_PRIORITY_QUEUE: {},
+    LOW_PRIORITY_QUEUE: {},
+    DEFAULT_PRIORITY_QUEUE: {}
+}
+
 ############################ APPS #####################################
 
 INSTALLED_APPS = (
@@ -249,8 +297,12 @@ INSTALLED_APPS = (
     'django.contrib.sessions',
     'django.contrib.sites',
     'django.contrib.messages',
+    'djcelery',
     'south',
 
+    # Monitor the status of services
+    'service_status',
+
     # For CMS
     'contentstore',
     'auth',
@@ -268,3 +320,7 @@ INSTALLED_APPS = (
     # comment common
     'django_comment_common',
 )
+
+################# EDX MARKETING SITE ##################################
+
+EDXMKTG_COOKIE_NAME = 'edxloggedin'

cms/envs/dev.py

@@ -116,6 +116,11 @@ SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'
 
 PIPELINE_SASS_ARGUMENTS = '--debug-info --require {proj_dir}/static/sass/bourbon/lib/bourbon.rb'.format(proj_dir=PROJECT_ROOT)
 
+################################# CELERY ######################################
+
+# By default don't use a worker, execute tasks as if they were local functions
+CELERY_ALWAYS_EAGER = True
+
 ################################ DEBUG TOOLBAR #################################
 INSTALLED_APPS += ('debug_toolbar', 'debug_toolbar_mongo')
 MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',)
@@ -150,5 +155,8 @@ DEBUG_TOOLBAR_MONGO_STACKTRACES = True
 # disable NPS survey in dev mode
 MITX_FEATURES['STUDIO_NPS_SURVEY'] = False
 
+# Enable URL that shows information about the status of variuous services
+MITX_FEATURES['ENABLE_SERVICE_STATUS'] = True
+
 # segment-io key for dev
 SEGMENT_IO_KEY = 'mty8edrrsg'

cms/envs/dev_with_worker.py

@@ -0,0 +1,35 @@
+"""
+This config file follows the dev enviroment, but adds the
+requirement of a celery worker running in the background to process
+celery tasks.
+
+The worker can be executed using:
+
+django_admin.py celery worker
+"""
+
+from dev import *
+
+################################# CELERY ######################################
+
+# Requires a separate celery worker
+
+CELERY_ALWAYS_EAGER = False
+
+# Use django db as the broker and result store
+
+BROKER_URL = 'django://'
+INSTALLED_APPS += ('djcelery.transport', )
+CELERY_RESULT_BACKEND = 'database'
+DJKOMBU_POLLING_INTERVAL = 1.0
+
+# Disable transaction management because we are using a worker. Views
+# that request a task and wait for the result will deadlock otherwise.
+
+MIDDLEWARE_CLASSES = tuple(
+    c for c in MIDDLEWARE_CLASSES
+    if c != 'django.middleware.transaction.TransactionMiddleware')
+
+# Note: other alternatives for disabling transactions don't work in 1.4
+# https://code.djangoproject.com/ticket/2304
+# https://code.djangoproject.com/ticket/16039

cms/envs/test.py

@@ -41,14 +41,14 @@ MODULESTORE_OPTIONS = {
     'default_class': 'xmodule.raw_module.RawDescriptor',
     'host': 'localhost',
     'db': 'test_xmodule',
-    'collection': 'modulestore',
-    'fs_root': GITHUB_REPO_ROOT,
+    'collection': 'test_modulestore',
+    'fs_root': TEST_ROOT / "data",
     'render_template': 'mitxmako.shortcuts.render_to_string',
 }
 
 MODULESTORE = {
     'default': {
-        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
+        'ENGINE': 'xmodule.modulestore.mongo.DraftMongoModuleStore',
         'OPTIONS': MODULESTORE_OPTIONS
     },
     'direct': {
@@ -108,6 +108,12 @@ CACHES = {
     }
 }
 
+################################# CELERY ######################################
+
+CELERY_ALWAYS_EAGER = True
+CELERY_RESULT_BACKEND = 'cache'
+BROKER_TRANSPORT = 'memory'
+
 ################### Make tests faster
 #http://slacy.com/blog/2012/04/make-your-tests-faster-in-django-1-4/
 PASSWORD_HASHERS = (
@@ -121,3 +127,4 @@ SEGMENT_IO_KEY = '***REMOVED***'
 # disable NPS survey in test mode
 MITX_FEATURES['STUDIO_NPS_SURVEY'] = False
 
+MITX_FEATURES['ENABLE_SERVICE_STATUS'] = True