Grade report celery task and direct file push to S3 from the new instructor dashboard.

Hook up display of grade files ready for download to new instructor dashboard.

LMS-58

lms/envs/aws.py

@@ -86,6 +86,7 @@ CELERY_DEFAULT_EXCHANGE = 'edx.{0}core'.format(QUEUE_VARIANT)
 HIGH_PRIORITY_QUEUE = 'edx.{0}core.high'.format(QUEUE_VARIANT)
 DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default'.format(QUEUE_VARIANT)
 LOW_PRIORITY_QUEUE = 'edx.{0}core.low'.format(QUEUE_VARIANT)
+HIGH_MEM_QUEUE = 'edx.{0}core.high_mem'.format(QUEUE_VARIANT)
 
 CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE
 CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE
@@ -93,9 +94,19 @@ CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE
 CELERY_QUEUES = {
     HIGH_PRIORITY_QUEUE: {},
     LOW_PRIORITY_QUEUE: {},
-    DEFAULT_PRIORITY_QUEUE: {}
+    DEFAULT_PRIORITY_QUEUE: {},
+    HIGH_MEM_QUEUE: {},
 }
 
+# If we're a worker on the high_mem queue, set ourselves to die after processing
+# one request to avoid having memory leaks take down the worker server. This env
+# var is set in /etc/init/edx-workers.conf -- this should probably be replaced
+# with some celery API call to see what queue we started listening to, but I
+# don't know what that call is or if it's active at this point in the code.
+if os.environ.get('QUEUE') == 'high_mem':
+    CELERYD_MAX_TASKS_PER_CHILD = 1
+
+
 ########################## NON-SECURE ENV CONFIG ##############################
 # Things like server locations, ports, etc.
 
@@ -312,3 +323,8 @@ TRACKING_BACKENDS.update(AUTH_TOKENS.get("TRACKING_BACKENDS", {}))
 
 # Student identity verification settings
 VERIFY_STUDENT = AUTH_TOKENS.get("VERIFY_STUDENT", VERIFY_STUDENT)
+
+# Grades download
+GRADES_DOWNLOAD_ROUTING_KEY = HIGH_MEM_QUEUE
+
+GRADES_DOWNLOAD = ENV_TOKENS.get("GRADES_DOWNLOAD", GRADES_DOWNLOAD)

lms/envs/common.py

@@ -192,6 +192,10 @@ MITX_FEATURES = {
     # Disable instructor dash buttons for downloading course data
     # when enrollment exceeds this number
     'MAX_ENROLLMENT_INSTR_BUTTONS': 200,
+
+    # Grade calculation started from the new instructor dashboard will write
+    # grades CSV files to S3 and give links for downloads.
+    'ENABLE_S3_GRADE_DOWNLOADS' : True,
 }
 
 # Used for A/B testing
@@ -846,6 +850,7 @@ CELERY_DEFAULT_EXCHANGE_TYPE = 'direct'
 HIGH_PRIORITY_QUEUE = 'edx.core.high'
 DEFAULT_PRIORITY_QUEUE = 'edx.core.default'
 LOW_PRIORITY_QUEUE = 'edx.core.low'
+HIGH_MEM_QUEUE = 'edx.core.high_mem'
 
 CELERY_QUEUE_HA_POLICY = 'all'
 
@@ -857,7 +862,8 @@ CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE
 CELERY_QUEUES = {
     HIGH_PRIORITY_QUEUE: {},
     LOW_PRIORITY_QUEUE: {},
-    DEFAULT_PRIORITY_QUEUE: {}
+    DEFAULT_PRIORITY_QUEUE: {},
+    HIGH_MEM_QUEUE: {},
 }
 
 # let logging work as configured:
@@ -1061,3 +1067,12 @@ REGISTRATION_OPTIONAL_FIELDS = set([
     'mailing_address',
     'goals',
 ])
+
+# Grades download
+GRADES_DOWNLOAD_ROUTING_KEY = HIGH_MEM_QUEUE
+
+GRADES_DOWNLOAD = {
+    'STORAGE_TYPE' : 'localfs',
+    'BUCKET' : 'edx-grades',
+    'ROOT_PATH' : '/tmp/edx-s3/grades',
+}

lms/envs/dev.py

@@ -283,6 +283,13 @@ EDX_API_KEY = None
 ####################### Shoppingcart ###########################
 MITX_FEATURES['ENABLE_SHOPPING_CART'] = True
 
+###################### Grade Downloads ######################
+GRADES_DOWNLOAD = {
+    'STORAGE_TYPE' : 'localfs',
+    'BUCKET' : 'edx-grades',
+    'ROOT_PATH' : '/tmp/edx-s3/grades',
+}
+
 #####################################################################
 # Lastly, see if the developer has any local overrides.
 try:

lms/envs/test.py

@@ -247,6 +247,13 @@ PASSWORD_HASHERS = (
     # 'django.contrib.auth.hashers.CryptPasswordHasher',
 )
 
+###################### Grade Downloads ######################
+GRADES_DOWNLOAD = {
+    'STORAGE_TYPE' : 'localfs',
+    'BUCKET' : 'edx-grades',
+    'ROOT_PATH' : '/tmp/edx-s3/grades',
+}
+
 ################### Make tests quieter
 
 # OpenID spews messages like this to stderr, we don't need to see them: