diff --git a/.gitignore b/.gitignore index 9c82bb8ea9..05e76c4caa 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ Gemfile.lock conf/locale/en/LC_MESSAGES/*.po !messages.po lms/static/sass/*.css +lms/static/sass/application.scss cms/static/sass/*.css lms/lib/comment_client/python nosetests.xml diff --git a/.reviewboardrc b/.reviewboardrc new file mode 100644 index 0000000000..b79235a4a4 --- /dev/null +++ b/.reviewboardrc @@ -0,0 +1,2 @@ +REVIEWBOARD_URL = "https://rbcommons.com/s/edx/" +GUESS_FIELDS = True diff --git a/cms/djangoapps/contentstore/views/component.py b/cms/djangoapps/contentstore/views/component.py index 34a659ab29..89b5e8bdc7 100644 --- a/cms/djangoapps/contentstore/views/component.py +++ b/cms/djangoapps/contentstore/views/component.py @@ -42,7 +42,7 @@ COMPONENT_TYPES = ['customtag', 'discussion', 'html', 'problem', 'video'] OPEN_ENDED_COMPONENT_TYPES = ["combinedopenended", "peergrading"] NOTE_COMPONENT_TYPES = ['notes'] -ADVANCED_COMPONENT_TYPES = ['annotatable' + 'word_cloud'] + OPEN_ENDED_COMPONENT_TYPES + NOTE_COMPONENT_TYPES +ADVANCED_COMPONENT_TYPES = ['annotatable', 'word_cloud'] + OPEN_ENDED_COMPONENT_TYPES + NOTE_COMPONENT_TYPES ADVANCED_COMPONENT_CATEGORY = 'advanced' ADVANCED_COMPONENT_POLICY_KEY = 'advanced_modules' diff --git a/cms/envs/acceptance.py b/cms/envs/acceptance.py index f4b867d3c6..36616ab257 100644 --- a/cms/envs/acceptance.py +++ b/cms/envs/acceptance.py @@ -2,6 +2,11 @@ This config file extends the test environment configuration so that we can run the lettuce acceptance tests. """ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .test import * # You need to start the server in debug mode, @@ -23,7 +28,7 @@ MODULESTORE_OPTIONS = { MODULESTORE = { 'default': { - 'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore', + 'ENGINE': 'xmodule.modulestore.mongo.DraftMongoModuleStore', 'OPTIONS': MODULESTORE_OPTIONS }, 'direct': { diff --git a/cms/envs/aws.py b/cms/envs/aws.py index 3cd70826da..9fabb3b9e8 100644 --- a/cms/envs/aws.py +++ b/cms/envs/aws.py @@ -1,6 +1,11 @@ """ This is the default template for our main set of AWS servers. """ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + import json from .common import * diff --git a/cms/envs/common.py b/cms/envs/common.py index e150374cef..038c00ddbb 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -19,6 +19,10 @@ Longer TODO: multiple sites, but we do need a way to map their data assets. """ +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + import sys import lms.envs.common from path import path diff --git a/cms/envs/dev.py b/cms/envs/dev.py index f3080c356f..203e4bd909 100644 --- a/cms/envs/dev.py +++ b/cms/envs/dev.py @@ -1,6 +1,10 @@ """ This config file runs the simplest dev environment""" +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .common import * from logsettings import get_logger_config diff --git a/cms/envs/dev_ike.py b/cms/envs/dev_ike.py index 1ebf219d44..0c798b68aa 100644 --- a/cms/envs/dev_ike.py +++ b/cms/envs/dev_ike.py @@ -1,3 +1,7 @@ +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + # dev environment for ichuang/mit # FORCE_SCRIPT_NAME = '/cms' diff --git a/cms/envs/dev_with_worker.py b/cms/envs/dev_with_worker.py index c5fc256ac9..078567c493 100644 --- a/cms/envs/dev_with_worker.py +++ b/cms/envs/dev_with_worker.py @@ -8,6 +8,10 @@ The worker can be executed using: django_admin.py celery worker """ +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from dev import * ################################# CELERY ###################################### diff --git a/cms/envs/jasmine.py b/cms/envs/jasmine.py index 6c7cbcdcb0..f3a982aa43 100644 --- a/cms/envs/jasmine.py +++ b/cms/envs/jasmine.py @@ -2,6 +2,10 @@ This configuration is used for running jasmine tests """ +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .test import * from logsettings import get_logger_config diff --git a/cms/envs/test.py b/cms/envs/test.py index 4cb975e2fb..6d78b0d7d6 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -7,6 +7,11 @@ sessions. Assumes structure: /mitx # The location of this repo /log # Where we're going to write log files """ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .common import * import os from path import path diff --git a/cms/static/sass/_mixins-inherited.scss b/cms/static/sass/_mixins-inherited.scss index a766919c74..c0d9df77ce 100644 --- a/cms/static/sass/_mixins-inherited.scss +++ b/cms/static/sass/_mixins-inherited.scss @@ -5,6 +5,12 @@ // talbs: we need to slowly ween ourselves off of these // ==================== + +// line-height (old way) +@function lh($amount: 1) { + @return $body-line-height * $amount; +} + // inherited - vertical and horizontal centering @mixin vertically-and-horizontally-centered ($height, $width) { left: 50%; diff --git a/cms/static/sass/elements/_typography.scss b/cms/static/sass/elements/_typography.scss index 9e9ca69652..69e78750c0 100644 --- a/cms/static/sass/elements/_typography.scss +++ b/cms/static/sass/elements/_typography.scss @@ -11,54 +11,54 @@ .t-title1 { @extend .t-title; @include font-size(60); - @include lh(60); + @include line-height(60); } .t-title2 { @extend .t-title; @include font-size(48); - @include lh(48); + @include line-height(48); } .t-title3 { @include font-size(36); - @include lh(36); + @include line-height(36); } .t-title4 { @extend .t-title; @include font-size(24); - @include lh(24); + @include line-height(24); } .t-title5 { @extend .t-title; @include font-size(18); - @include lh(18); + @include line-height(18); } .t-title6 { @extend .t-title; @include font-size(16); - @include lh(16); + @include line-height(16); } .t-title7 { @extend .t-title; @include font-size(14); - @include lh(14); + @include line-height(14); } .t-title8 { @extend .t-title; @include font-size(12); - @include lh(12); + @include line-height(12); } .t-title9 { @extend .t-title; @include font-size(11); - @include lh(11); + @include line-height(11); } // ==================== @@ -71,31 +71,31 @@ .t-copy-base { @extend .t-copy; @include font-size(16); - @include lh(16); + @include line-height(16); } .t-copy-lead1 { @extend .t-copy; @include font-size(18); - @include lh(18); + @include line-height(18); } .t-copy-lead2 { @extend .t-copy; @include font-size(24); - @include lh(24); + @include line-height(24); } .t-copy-sub1 { @extend .t-copy; @include font-size(14); - @include lh(14); + @include line-height(14); } .t-copy-sub2 { @extend .t-copy; @include font-size(12); - @include lh(12); + @include line-height(12); } // ==================== @@ -103,22 +103,22 @@ // actions/labels .t-action1 { @include font-size(18); - @include lh(18); + @include line-height(18); } .t-action2 { @include font-size(16); - @include lh(16); + @include line-height(16); } .t-action3 { @include font-size(14); - @include lh(14); + @include line-height(14); } .t-action4 { @include font-size(12); - @include lh(12); + @include line-height(12); } diff --git a/common/lib/calc/setup.py b/common/lib/calc/setup.py index f7bb1708af..cb638914f9 100644 --- a/common/lib/calc/setup.py +++ b/common/lib/calc/setup.py @@ -2,7 +2,7 @@ from setuptools import setup setup( name="calc", - version="0.1", + version="0.1.1", py_modules=["calc"], install_requires=[ "pyparsing==1.5.6", diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 7ead599d67..8543e9e3e1 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -475,12 +475,11 @@ class LoncapaProblem(object): msg = "Error while executing script code: %s" % str(err).replace('<', '<') raise responsetypes.LoncapaProblemError(msg) - # store code source in context + # Store code source in context, along with the Python path needed to run it correctly. context['script_code'] = all_code + context['python_path'] = python_path return context - - def _extract_html(self, problemtree): # private ''' Main (private) function which converts Problem XML tree to HTML. diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index c7a99f1271..a166438f17 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -286,7 +286,7 @@ class LoncapaResponse(object): } try: - safe_exec.safe_exec(code, globals_dict) + safe_exec.safe_exec(code, globals_dict, python_path=self.context['python_path']) except Exception as err: msg = 'Error %s in evaluating hint function %s' % (err, hintfn) msg += "\nSee XML source line %s" % getattr( @@ -972,7 +972,7 @@ class CustomResponse(LoncapaResponse): 'ans': ans, } globals_dict.update(kwargs) - safe_exec.safe_exec(code, globals_dict, cache=self.system.cache) + safe_exec.safe_exec(code, globals_dict, python_path=self.context['python_path']) return globals_dict['cfn_return'] return check_function diff --git a/common/lib/capa/capa/safe_exec/tests/test_lazymod.py b/common/lib/capa/capa/safe_exec/tests/test_lazymod.py index 68dcd81ea7..6a8ed5ff48 100644 --- a/common/lib/capa/capa/safe_exec/tests/test_lazymod.py +++ b/common/lib/capa/capa/safe_exec/tests/test_lazymod.py @@ -39,6 +39,9 @@ class TestLazyMod(unittest.TestCase): self.assertEqual(hsv[0], 0.25) def test_dotted(self): - self.assertNotIn("email.utils", sys.modules) - email_utils = LazyModule("email.utils") - self.assertEqual(email_utils.quote('"hi"'), r'\"hi\"') + # wsgiref is a module with submodules that is not already imported. + # Any similar module would do. This test demonstrates that the module + # is not already im + self.assertNotIn("wsgiref.util", sys.modules) + wsgiref_util = LazyModule("wsgiref.util") + self.assertEqual(wsgiref_util.guess_scheme({}), "http") diff --git a/common/lib/chem/setup.py b/common/lib/chem/setup.py index 4f2b24ddee..642c9a4fe5 100644 --- a/common/lib/chem/setup.py +++ b/common/lib/chem/setup.py @@ -2,7 +2,7 @@ from setuptools import setup setup( name="chem", - version="0.1", + version="0.1.1", packages=["chem"], install_requires=[ "pyparsing==1.5.6", diff --git a/common/lib/sandbox-packages/setup.py b/common/lib/sandbox-packages/setup.py index 1b99118aca..96c1190e38 100644 --- a/common/lib/sandbox-packages/setup.py +++ b/common/lib/sandbox-packages/setup.py @@ -2,7 +2,7 @@ from setuptools import setup setup( name="sandbox-packages", - version="0.1", + version="0.1.1", packages=[ "verifiers", ], diff --git a/common/lib/xmodule/xmodule/combined_open_ended_module.py b/common/lib/xmodule/xmodule/combined_open_ended_module.py index f4074283fe..a2ce4d0a65 100644 --- a/common/lib/xmodule/xmodule/combined_open_ended_module.py +++ b/common/lib/xmodule/xmodule/combined_open_ended_module.py @@ -8,7 +8,7 @@ from .x_module import XModule from xblock.core import Integer, Scope, String, Boolean, List from xmodule.open_ended_grading_classes.combined_open_ended_modulev1 import CombinedOpenEndedV1Module, CombinedOpenEndedV1Descriptor from collections import namedtuple -from .fields import Date, StringyFloat +from .fields import Date, StringyFloat, StringyInteger, StringyBoolean log = logging.getLogger("mitx.courseware") @@ -49,19 +49,19 @@ class VersionInteger(Integer): class CombinedOpenEndedFields(object): display_name = String(help="Display name for this module", default="Open Ended Grading", scope=Scope.settings) - current_task_number = Integer(help="Current task that the student is on.", default=0, scope=Scope.user_state) + current_task_number = StringyInteger(help="Current task that the student is on.", default=0, scope=Scope.user_state) task_states = List(help="List of state dictionaries of each task within this module.", scope=Scope.user_state) state = String(help="Which step within the current task that the student is on.", default="initial", scope=Scope.user_state) - student_attempts = Integer(help="Number of attempts taken by the student on this problem", default=0, + student_attempts = StringyInteger(help="Number of attempts taken by the student on this problem", default=0, scope=Scope.user_state) - ready_to_reset = Boolean(help="If the problem is ready to be reset or not.", default=False, + ready_to_reset = StringyBoolean(help="If the problem is ready to be reset or not.", default=False, scope=Scope.user_state) - attempts = Integer(help="Maximum number of attempts that a student is allowed.", default=1, scope=Scope.settings) - is_graded = Boolean(help="Whether or not the problem is graded.", default=False, scope=Scope.settings) - accept_file_upload = Boolean(help="Whether or not the problem accepts file uploads.", default=False, + attempts = StringyInteger(help="Maximum number of attempts that a student is allowed.", default=1, scope=Scope.settings) + is_graded = StringyBoolean(help="Whether or not the problem is graded.", default=False, scope=Scope.settings) + accept_file_upload = StringyBoolean(help="Whether or not the problem accepts file uploads.", default=False, scope=Scope.settings) - skip_spelling_checks = Boolean(help="Whether or not to skip initial spelling checks.", default=True, + skip_spelling_checks = StringyBoolean(help="Whether or not to skip initial spelling checks.", default=True, scope=Scope.settings) due = Date(help="Date that this problem is due by", default=None, scope=Scope.settings) graceperiod = String(help="Amount of time after the due date that submissions will be accepted", default=None, diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py index 114a3281c6..7128c04a88 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml.py +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -290,7 +290,6 @@ class XMLModuleStore(ModuleStoreBase): if course_dirs is None: course_dirs = sorted([d for d in os.listdir(self.data_dir) if os.path.exists(self.data_dir / d / "course.xml")]) - for course_dir in course_dirs: self.try_load_course(course_dir) diff --git a/common/lib/xmodule/xmodule/open_ended_grading_classes/controller_query_service.py b/common/lib/xmodule/xmodule/open_ended_grading_classes/controller_query_service.py index 08f2a95387..b807e05160 100644 --- a/common/lib/xmodule/xmodule/open_ended_grading_classes/controller_query_service.py +++ b/common/lib/xmodule/xmodule/open_ended_grading_classes/controller_query_service.py @@ -6,7 +6,7 @@ log = logging.getLogger(__name__) class ControllerQueryService(GradingService): """ - Interface to staff grading backend. + Interface to controller query backend. """ def __init__(self, config, system): @@ -77,6 +77,50 @@ class ControllerQueryService(GradingService): return response +class MockControllerQueryService(object): + """ + Mock controller query service for testing + """ + + def __init__(self, config, system): + pass + + def check_if_name_is_unique(self, **params): + """ + Mock later if needed. Stub function for now. + @param params: + @return: + """ + pass + + def check_for_eta(self, **params): + """ + Mock later if needed. Stub function for now. + @param params: + @return: + """ + pass + + def check_combined_notifications(self, **params): + combined_notifications = '{"flagged_submissions_exist": false, "version": 1, "new_student_grading_to_view": false, "success": true, "staff_needs_to_grade": false, "student_needs_to_peer_grade": true, "overall_need_to_check": true}' + return combined_notifications + + def get_grading_status_list(self, **params): + grading_status_list = '{"version": 1, "problem_list": [{"problem_name": "Science Question -- Machine Assessed", "grader_type": "NA", "eta_available": true, "state": "Waiting to be Graded", "eta": 259200, "location": "i4x://MITx/oe101x/combinedopenended/Science_SA_ML"}, {"problem_name": "Humanities Question -- Peer Assessed", "grader_type": "NA", "eta_available": true, "state": "Waiting to be Graded", "eta": 259200, "location": "i4x://MITx/oe101x/combinedopenended/Humanities_SA_Peer"}], "success": true}' + return grading_status_list + + def get_flagged_problem_list(self, **params): + flagged_problem_list = '{"version": 1, "success": false, "error": "No flagged submissions exist for course: MITx/oe101x/2012_Fall"}' + return flagged_problem_list + + def take_action_on_flags(self, **params): + """ + Mock later if needed. Stub function for now. + @param params: + @return: + """ + pass + def convert_seconds_to_human_readable(seconds): if seconds < 60: human_string = "{0} seconds".format(seconds) diff --git a/common/static/sass/_mixins.scss b/common/static/sass/_mixins.scss index 6e1a34aaaa..c3a548bbf7 100644 --- a/common/static/sass/_mixins.scss +++ b/common/static/sass/_mixins.scss @@ -8,7 +8,7 @@ } // mixins - line height -@mixin lh($fontSize: auto){ +@mixin line-height($fontSize: auto){ line-height: ($fontSize*1.48) + px; line-height: (($fontSize/10)*1.48) + rem; } diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index d6c104a83c..2a665cd8a0 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -414,6 +414,11 @@ def modx_dispatch(request, dispatch, location, course_id): through the part before the first '?'. - location -- the module location. Used to look up the XModule instance - course_id -- defines the course context for this request. + + Raises PermissionDenied if the user is not logged in. Raises Http404 if + the location and course_id do not identify a valid module, the module is + not accessible by the user, or the module raises NotFoundError. If the + module raises any other error, it will escape this function. ''' # ''' (fix emacs broken parsing) @@ -442,8 +447,19 @@ def modx_dispatch(request, dispatch, location, course_id): return HttpResponse(json.dumps({'success': file_too_big_msg})) p[fileinput_id] = inputfiles + try: + descriptor = modulestore().get_instance(course_id, location) + except ItemNotFoundError: + log.warn( + "Invalid location for course id {course_id}: {location}".format( + course_id=course_id, + location=location + ) + ) + raise Http404 + model_data_cache = ModelDataCache.cache_for_descriptor_descendents(course_id, - request.user, modulestore().get_instance(course_id, location)) + request.user, descriptor) instance = get_module(request.user, request, location, model_data_cache, course_id, grade_bucket_type='ajax') if instance is None: diff --git a/lms/djangoapps/courseware/tests/test_module_render.py b/lms/djangoapps/courseware/tests/test_module_render.py index 0e4ba8ba5e..94ab4b7e94 100644 --- a/lms/djangoapps/courseware/tests/test_module_render.py +++ b/lms/djangoapps/courseware/tests/test_module_render.py @@ -69,19 +69,38 @@ class ModuleRenderTestCase(LoginEnrollmentTestCase): json.dumps({'success': 'Submission aborted! Your file "%s" is too large (max size: %d MB)' % (inputfile.name, settings.STUDENT_FILEUPLOAD_MAX_SIZE / (1000 ** 2))})) mock_request_3 = MagicMock() - mock_request_3.POST.copy.return_value = {} + mock_request_3.POST.copy.return_value = {'position': 1} mock_request_3.FILES = False mock_request_3.user = UserFactory() inputfile_2 = Stub() inputfile_2.size = 1 inputfile_2.name = 'name' - self.assertRaises(ItemNotFoundError, render.modx_dispatch, - mock_request_3, 'dummy', self.location, 'toy') - self.assertRaises(Http404, render.modx_dispatch, mock_request_3, 'dummy', - self.location, self.course_id) - mock_request_3.POST.copy.return_value = {'position': 1} self.assertIsInstance(render.modx_dispatch(mock_request_3, 'goto_position', self.location, self.course_id), HttpResponse) + self.assertRaises( + Http404, + render.modx_dispatch, + mock_request_3, + 'goto_position', + self.location, + 'bad_course_id' + ) + self.assertRaises( + Http404, + render.modx_dispatch, + mock_request_3, + 'goto_position', + ['i4x', 'edX', 'toy', 'chapter', 'bad_location'], + self.course_id + ) + self.assertRaises( + Http404, + render.modx_dispatch, + mock_request_3, + 'bad_dispatch', + self.location, + self.course_id + ) def test_get_score_bucket(self): self.assertEquals(render.get_score_bucket(0, 10), 'incorrect') diff --git a/lms/djangoapps/django_comment_client/forum/views.py b/lms/djangoapps/django_comment_client/forum/views.py index 55797227ea..b04bd787d8 100644 --- a/lms/djangoapps/django_comment_client/forum/views.py +++ b/lms/djangoapps/django_comment_client/forum/views.py @@ -174,6 +174,9 @@ def forum_form_discussion(request, course_id): try: unsafethreads, query_params = get_threads(request, course_id) # This might process a search query threads = [utils.safe_content(thread) for thread in unsafethreads] + except (cc.utils.CommentClientMaintenanceError) as err: + log.warning("Forum is in maintenance mode") + return render_to_response('discussion/maintenance.html', {}) except (cc.utils.CommentClientError, cc.utils.CommentClientUnknownError) as err: log.error("Error loading forum discussion threads: %s" % str(err)) raise Http404 diff --git a/lms/djangoapps/notes/migrations/0001_initial.py b/lms/djangoapps/notes/migrations/0001_initial.py index 0372a889df..1629b2355d 100644 --- a/lms/djangoapps/notes/migrations/0001_initial.py +++ b/lms/djangoapps/notes/migrations/0001_initial.py @@ -13,7 +13,7 @@ class Migration(SchemaMigration): ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), ('course_id', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)), - ('uri', self.gf('django.db.models.fields.CharField')(max_length=512, db_index=True)), + ('uri', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)), ('text', self.gf('django.db.models.fields.TextField')(default='')), ('quote', self.gf('django.db.models.fields.TextField')(default='')), ('range_start', self.gf('django.db.models.fields.CharField')(max_length=2048)), @@ -82,7 +82,7 @@ class Migration(SchemaMigration): 'tags': ('django.db.models.fields.TextField', [], {'default': "''"}), 'text': ('django.db.models.fields.TextField', [], {'default': "''"}), 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}), - 'uri': ('django.db.models.fields.CharField', [], {'max_length': '512', 'db_index': 'True'}), + 'uri': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) } } diff --git a/lms/djangoapps/notes/models.py b/lms/djangoapps/notes/models.py index 4050516f13..aa2ec7a377 100644 --- a/lms/djangoapps/notes/models.py +++ b/lms/djangoapps/notes/models.py @@ -9,7 +9,7 @@ import json class Note(models.Model): user = models.ForeignKey(User, db_index=True) course_id = models.CharField(max_length=255, db_index=True) - uri = models.CharField(max_length=512, db_index=True) + uri = models.CharField(max_length=255, db_index=True) text = models.TextField(default="") quote = models.TextField(default="") range_start = models.CharField(max_length=2048) # xpath string diff --git a/lms/djangoapps/open_ended_grading/tests.py b/lms/djangoapps/open_ended_grading/tests.py index ffc02608d5..18ba863d69 100644 --- a/lms/djangoapps/open_ended_grading/tests.py +++ b/lms/djangoapps/open_ended_grading/tests.py @@ -5,19 +5,20 @@ django-admin.py test --settings=lms.envs.test --pythonpath=. lms/djangoapps/open """ import json -from mock import MagicMock +from mock import MagicMock, patch, Mock from django.core.urlresolvers import reverse from django.contrib.auth.models import Group +from django.http import HttpResponse from mitxmako.shortcuts import render_to_string -from xmodule.open_ended_grading_classes import peer_grading_service +from xmodule.open_ended_grading_classes import peer_grading_service, controller_query_service from xmodule import peer_grading_module from xmodule.modulestore.django import modulestore import xmodule.modulestore.django from xmodule.x_module import ModuleSystem -from open_ended_grading import staff_grading_service +from open_ended_grading import staff_grading_service, views from courseware.access import _course_staff_group_name from courseware.tests.tests import LoginEnrollmentTestCase, TEST_DATA_XML_MODULESTORE, get_user @@ -25,10 +26,11 @@ import logging log = logging.getLogger(__name__) from django.test.utils import override_settings -from django.http import QueryDict from xmodule.tests import test_util_open_ended +from courseware.tests import factories + @override_settings(MODULESTORE=TEST_DATA_XML_MODULESTORE) class TestStaffGradingService(LoginEnrollmentTestCase): @@ -55,8 +57,8 @@ class TestStaffGradingService(LoginEnrollmentTestCase): def make_instructor(course): group_name = _course_staff_group_name(course.location) - g = Group.objects.create(name=group_name) - g.user_set.add(get_user(self.instructor)) + group = Group.objects.create(name=group_name) + group.user_set.add(get_user(self.instructor)) make_instructor(self.toy) @@ -76,30 +78,28 @@ class TestStaffGradingService(LoginEnrollmentTestCase): self.check_for_get_code(404, url) self.check_for_post_code(404, url) - def test_get_next(self): self.login(self.instructor, self.password) url = reverse('staff_grading_get_next', kwargs={'course_id': self.course_id}) data = {'location': self.location} - r = self.check_for_post_code(200, url, data) + response = self.check_for_post_code(200, url, data) - d = json.loads(r.content) + content = json.loads(response.content) - self.assertTrue(d['success']) - self.assertEquals(d['submission_id'], self.mock_service.cnt) - self.assertIsNotNone(d['submission']) - self.assertIsNotNone(d['num_graded']) - self.assertIsNotNone(d['min_for_ml']) - self.assertIsNotNone(d['num_pending']) - self.assertIsNotNone(d['prompt']) - self.assertIsNotNone(d['ml_error_info']) - self.assertIsNotNone(d['max_score']) - self.assertIsNotNone(d['rubric']) + self.assertTrue(content['success']) + self.assertEquals(content['submission_id'], self.mock_service.cnt) + self.assertIsNotNone(content['submission']) + self.assertIsNotNone(content['num_graded']) + self.assertIsNotNone(content['min_for_ml']) + self.assertIsNotNone(content['num_pending']) + self.assertIsNotNone(content['prompt']) + self.assertIsNotNone(content['ml_error_info']) + self.assertIsNotNone(content['max_score']) + self.assertIsNotNone(content['rubric']) - - def save_grade_base(self,skip=False): + def save_grade_base(self, skip=False): self.login(self.instructor, self.password) url = reverse('staff_grading_save_grade', kwargs={'course_id': self.course_id}) @@ -111,12 +111,12 @@ class TestStaffGradingService(LoginEnrollmentTestCase): 'submission_flagged': "true", 'rubric_scores[]': ['1', '2']} if skip: - data.update({'skipped' : True}) + data.update({'skipped': True}) - r = self.check_for_post_code(200, url, data) - d = json.loads(r.content) - self.assertTrue(d['success'], str(d)) - self.assertEquals(d['submission_id'], self.mock_service.cnt) + response = self.check_for_post_code(200, url, data) + content = json.loads(response.content) + self.assertTrue(content['success'], str(content)) + self.assertEquals(content['submission_id'], self.mock_service.cnt) def test_save_grade(self): self.save_grade_base(skip=False) @@ -130,11 +130,11 @@ class TestStaffGradingService(LoginEnrollmentTestCase): url = reverse('staff_grading_get_problem_list', kwargs={'course_id': self.course_id}) data = {} - r = self.check_for_post_code(200, url, data) - d = json.loads(r.content) + response = self.check_for_post_code(200, url, data) + content = json.loads(response.content) - self.assertTrue(d['success'], str(d)) - self.assertIsNotNone(d['problem_list']) + self.assertTrue(content['success'], str(content)) + self.assertIsNotNone(content['problem_list']) @override_settings(MODULESTORE=TEST_DATA_XML_MODULESTORE) @@ -181,14 +181,14 @@ class TestPeerGradingService(LoginEnrollmentTestCase): def test_get_next_submission_success(self): data = {'location': self.location} - r = self.peer_module.get_next_submission(data) - d = r + response = self.peer_module.get_next_submission(data) + content = response - self.assertTrue(d['success']) - self.assertIsNotNone(d['submission_id']) - self.assertIsNotNone(d['prompt']) - self.assertIsNotNone(d['submission_key']) - self.assertIsNotNone(d['max_score']) + self.assertTrue(content['success']) + self.assertIsNotNone(content['submission_id']) + self.assertIsNotNone(content['prompt']) + self.assertIsNotNone(content['submission_key']) + self.assertIsNotNone(content['max_score']) def test_get_next_submission_missing_location(self): data = {} @@ -216,10 +216,9 @@ class TestPeerGradingService(LoginEnrollmentTestCase): qdict.getlist = fake_get_item qdict.keys = data.keys - r = self.peer_module.save_grade(qdict) - d = r + response = self.peer_module.save_grade(qdict) - self.assertTrue(d['success']) + self.assertTrue(response['success']) def test_save_grade_missing_keys(self): data = {} @@ -229,37 +228,35 @@ class TestPeerGradingService(LoginEnrollmentTestCase): def test_is_calibrated_success(self): data = {'location': self.location} - r = self.peer_module.is_student_calibrated(data) - d = r + response = self.peer_module.is_student_calibrated(data) - self.assertTrue(d['success']) - self.assertTrue('calibrated' in d) + self.assertTrue(response['success']) + self.assertTrue('calibrated' in response) def test_is_calibrated_failure(self): data = {} - d = self.peer_module.is_student_calibrated(data) - self.assertFalse(d['success']) - self.assertFalse('calibrated' in d) + response = self.peer_module.is_student_calibrated(data) + self.assertFalse(response['success']) + self.assertFalse('calibrated' in response) def test_show_calibration_essay_success(self): data = {'location': self.location} - r = self.peer_module.show_calibration_essay(data) - d = r + response = self.peer_module.show_calibration_essay(data) - self.assertTrue(d['success']) - self.assertIsNotNone(d['submission_id']) - self.assertIsNotNone(d['prompt']) - self.assertIsNotNone(d['submission_key']) - self.assertIsNotNone(d['max_score']) + self.assertTrue(response['success']) + self.assertIsNotNone(response['submission_id']) + self.assertIsNotNone(response['prompt']) + self.assertIsNotNone(response['submission_key']) + self.assertIsNotNone(response['max_score']) def test_show_calibration_essay_missing_key(self): data = {} - d = self.peer_module.show_calibration_essay(data) + response = self.peer_module.show_calibration_essay(data) - self.assertFalse(d['success']) - self.assertEqual(d['error'], "Missing required keys: location") + self.assertFalse(response['success']) + self.assertEqual(response['error'], "Missing required keys: location") def test_save_calibration_essay_success(self): data = { @@ -281,13 +278,45 @@ class TestPeerGradingService(LoginEnrollmentTestCase): qdict.getlist = fake_get_item qdict.keys = data.keys - d = self.peer_module.save_calibration_essay(qdict) - self.assertTrue(d['success']) - self.assertTrue('actual_score' in d) + response = self.peer_module.save_calibration_essay(qdict) + self.assertTrue(response['success']) + self.assertTrue('actual_score' in response) def test_save_calibration_essay_missing_keys(self): data = {} - d = self.peer_module.save_calibration_essay(data) - self.assertFalse(d['success']) - self.assertTrue(d['error'].find('Missing required keys:') > -1) - self.assertFalse('actual_score' in d) + response = self.peer_module.save_calibration_essay(data) + self.assertFalse(response['success']) + self.assertTrue(response['error'].find('Missing required keys:') > -1) + self.assertFalse('actual_score' in response) + + +@override_settings(MODULESTORE=TEST_DATA_XML_MODULESTORE) +class TestPanel(LoginEnrollmentTestCase): + """ + Run tests on the open ended panel + """ + + def setUp(self): + # Toy courses should be loaded + self.course_name = 'edX/open_ended/2012_Fall' + self.course = modulestore().get_course(self.course_name) + self.user = factories.UserFactory() + + def test_open_ended_panel(self): + """ + Test to see if the peer grading module in the demo course is found + @return: + """ + found_module, peer_grading_module = views.find_peer_grading_module(self.course) + self.assertTrue(found_module) + + @patch('xmodule.open_ended_grading_classes.controller_query_service.ControllerQueryService', + controller_query_service.MockControllerQueryService) + def test_problem_list(self): + """ + Ensure that the problem list from the grading controller server can be rendered properly locally + @return: + """ + request = Mock(user=self.user) + response = views.student_problem_list(request, self.course.id) + self.assertTrue(isinstance(response, HttpResponse)) diff --git a/lms/djangoapps/open_ended_grading/views.py b/lms/djangoapps/open_ended_grading/views.py index cb617d609d..2bb6f61491 100644 --- a/lms/djangoapps/open_ended_grading/views.py +++ b/lms/djangoapps/open_ended_grading/views.py @@ -21,6 +21,7 @@ import open_ended_notifications from xmodule.modulestore.django import modulestore from xmodule.modulestore import search +from xmodule.modulestore.exceptions import ItemNotFoundError from django.http import HttpResponse, Http404, HttpResponseRedirect from mitxmako.shortcuts import render_to_string @@ -30,10 +31,10 @@ log = logging.getLogger(__name__) system = ModuleSystem( ajax_url=None, track_function=None, - get_module = None, + get_module=None, render_template=render_to_string, - replace_urls = None, - xblock_model_data= {} + replace_urls=None, + xblock_model_data={} ) controller_qs = ControllerQueryService(settings.OPEN_ENDED_GRADING_INTERFACE, system) @@ -90,40 +91,61 @@ def staff_grading(request, course_id): 'staff_access': True, }) -@cache_control(no_cache=True, no_store=True, must_revalidate=True) -def peer_grading(request, course_id): - ''' - Show a peer grading interface - ''' - - #Get the current course - course = get_course_with_access(request.user, course_id, 'load') - course_id_parts = course.id.split("/") - false_dict = [False, "False", "false", "FALSE"] - +def find_peer_grading_module(course): + """ + Given a course, finds the first peer grading module in it. + @param course: A course object. + @return: boolean found_module, string problem_url + """ #Reverse the base course url base_course_url = reverse('courses') - try: - #TODO: This will not work with multiple runs of a course. Make it work. The last key in the Location passed - #to get_items is called revision. Is this the same as run? - #Get the peer grading modules currently in the course - items = modulestore().get_items(['i4x', None, course_id_parts[1], 'peergrading', None]) - #See if any of the modules are centralized modules (ie display info from multiple problems) - items = [i for i in items if getattr(i,"use_for_single_location", True) in false_dict] - #Get the first one + found_module = False + problem_url = "" + + #Get the course id and split it + course_id_parts = course.id.split("/") + log.info("COURSE ID PARTS") + log.info(course_id_parts) + #Get the peer grading modules currently in the course. Explicitly specify the course id to avoid issues with different runs. + items = modulestore().get_items(['i4x', course_id_parts[0], course_id_parts[1], 'peergrading', None], + course_id=course.id) + #See if any of the modules are centralized modules (ie display info from multiple problems) + items = [i for i in items if not getattr(i, "use_for_single_location", True)] + #Get the first one + if len(items) > 0: item_location = items[0].location #Generate a url for the first module and redirect the user to it problem_url_parts = search.path_to_location(modulestore(), course.id, item_location) problem_url = generate_problem_url(problem_url_parts, base_course_url) + found_module = True - return HttpResponseRedirect(problem_url) - except: + return found_module, problem_url + + +@cache_control(no_cache=True, no_store=True, must_revalidate=True) +def peer_grading(request, course_id): + ''' + When a student clicks on the "peer grading" button in the open ended interface, link them to a peer grading + xmodule in the course. + ''' + + #Get the current course + course = get_course_with_access(request.user, course_id, 'load') + + found_module, problem_url = find_peer_grading_module(course) + if not found_module: #This is a student_facing_error - error_message = "Error with initializing peer grading. Centralized module does not exist. Please contact course staff." + error_message = """ + Error with initializing peer grading. + There has not been a peer grading module created in the courseware that would allow you to grade others. + Please check back later for this. + """ #This is a dev_facing_error log.exception(error_message + "Current course is: {0}".format(course_id)) return HttpResponse(error_message) + return HttpResponseRedirect(problem_url) + def generate_problem_url(problem_url_parts, base_course_url): """ @@ -145,7 +167,8 @@ def generate_problem_url(problem_url_parts, base_course_url): @cache_control(no_cache=True, no_store=True, must_revalidate=True) def student_problem_list(request, course_id): ''' - Show a student problem list + Show a student problem list to a student. Fetch the list from the grading controller server, get some metadata, + and then show it to the student. ''' course = get_course_with_access(request.user, course_id, 'load') student_id = unique_id_for_user(request.user) @@ -157,6 +180,7 @@ def student_problem_list(request, course_id): base_course_url = reverse('courses') try: + #Get list of all open ended problems that the grading server knows about problem_list_json = controller_qs.get_grading_status_list(course_id, unique_id_for_user(request.user)) problem_list_dict = json.loads(problem_list_json) success = problem_list_dict['success'] @@ -166,8 +190,22 @@ def student_problem_list(request, course_id): else: problem_list = problem_list_dict['problem_list'] + #A list of problems to remove (problems that can't be found in the course) + list_to_remove = [] for i in xrange(0, len(problem_list)): - problem_url_parts = search.path_to_location(modulestore(), course.id, problem_list[i]['location']) + try: + #Try to load each problem in the courseware to get links to them + problem_url_parts = search.path_to_location(modulestore(), course.id, problem_list[i]['location']) + except ItemNotFoundError: + #If the problem cannot be found at the location received from the grading controller server, it has been deleted by the course author. + #Continue with the rest of the location to construct the list + error_message = "Could not find module for course {0} at location {1}".format(course.id, + problem_list[i][ + 'location']) + log.error(error_message) + #Mark the problem for removal from the list + list_to_remove.append(i) + continue problem_url = generate_problem_url(problem_url_parts, base_course_url) problem_list[i].update({'actual_url': problem_url}) eta_available = problem_list[i]['eta_available'] @@ -197,6 +235,8 @@ def student_problem_list(request, course_id): log.error("Problem with results from external grading service for open ended.") success = False + #Remove problems that cannot be found in the courseware from the list + problem_list = [problem_list[i] for i in xrange(0, len(problem_list)) if i not in list_to_remove] ajax_url = _reverse_with_slash('open_ended_problems', course_id) return render_to_response('open_ended_problems/open_ended_problems.html', { @@ -300,7 +340,16 @@ def combined_notifications(request, course_id): 'description': description, 'alert_message': alert_message } - notification_list.append(notification_item) + #The open ended panel will need to link the "peer grading" button in the panel to a peer grading + #xmodule defined in the course. This checks to see if the human name of the server notification + #that we are currently processing is "peer grading". If it is, it looks for a peer grading + #module in the course. If none exists, it removes the peer grading item from the panel. + if human_name == "Peer Grading": + found_module, problem_url = find_peer_grading_module(course) + if found_module: + notification_list.append(notification_item) + else: + notification_list.append(notification_item) ajax_url = _reverse_with_slash('open_ended_notifications', course_id) combined_dict = { @@ -311,9 +360,7 @@ def combined_notifications(request, course_id): 'ajax_url': ajax_url, } - return render_to_response('open_ended_problems/combined_notifications.html', - combined_dict - ) + return render_to_response('open_ended_problems/combined_notifications.html', combined_dict) @cache_control(no_cache=True, no_store=True, must_revalidate=True) diff --git a/lms/envs/acceptance.py b/lms/envs/acceptance.py index 611c3fdac8..3b87bb4326 100644 --- a/lms/envs/acceptance.py +++ b/lms/envs/acceptance.py @@ -2,6 +2,11 @@ This config file extends the test environment configuration so that we can run the lettuce acceptance tests. """ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .test import * # You need to start the server in debug mode, diff --git a/lms/envs/aws.py b/lms/envs/aws.py index bec2671d5e..a3d5cb653f 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -6,6 +6,11 @@ Common traits: * Use memcached, and cache-backed sessions * Use a MySQL 5.1 database """ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + import json from .common import * @@ -109,6 +114,11 @@ DEFAULT_FEEDBACK_EMAIL = ENV_TOKENS.get('DEFAULT_FEEDBACK_EMAIL', DEFAULT_FEEDBA ADMINS = ENV_TOKENS.get('ADMINS', ADMINS) SERVER_EMAIL = ENV_TOKENS.get('SERVER_EMAIL', SERVER_EMAIL) +#Theme overrides +THEME_NAME = ENV_TOKENS.get('THEME_NAME', None) +if not THEME_NAME is None: + enable_theme(THEME_NAME) + #Timezone overrides TIME_ZONE = ENV_TOKENS.get('TIME_ZONE', TIME_ZONE) diff --git a/lms/envs/cms/acceptance.py b/lms/envs/cms/acceptance.py index e5ee2937f4..0b638dca8a 100644 --- a/lms/envs/cms/acceptance.py +++ b/lms/envs/cms/acceptance.py @@ -3,6 +3,11 @@ This config file is a copy of dev environment without the Debug Toolbar. I it suitable to run against acceptance tests. """ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .dev import * # REMOVE DEBUG TOOLBAR diff --git a/lms/envs/cms/aws.py b/lms/envs/cms/aws.py index a0e2f25d83..baeaebca1c 100644 --- a/lms/envs/cms/aws.py +++ b/lms/envs/cms/aws.py @@ -2,6 +2,10 @@ Settings for the LMS that runs alongside the CMS on AWS """ +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from ..aws import * with open(ENV_ROOT / "cms.auth.json") as auth_file: diff --git a/lms/envs/cms/dev.py b/lms/envs/cms/dev.py index 9333b7883c..e55c6d61b5 100644 --- a/lms/envs/cms/dev.py +++ b/lms/envs/cms/dev.py @@ -2,6 +2,10 @@ Settings for the LMS that runs alongside the CMS on AWS """ +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from ..dev import * MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = False diff --git a/lms/envs/cms/preview_dev.py b/lms/envs/cms/preview_dev.py index 463af34624..1cfaec6159 100644 --- a/lms/envs/cms/preview_dev.py +++ b/lms/envs/cms/preview_dev.py @@ -2,6 +2,10 @@ Settings for the LMS that runs alongside the CMS on AWS """ +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .dev import * MODULESTORE = { diff --git a/lms/envs/common.py b/lms/envs/common.py index a198f010c6..741d624ed7 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -18,6 +18,11 @@ Longer TODO: 3. We need to handle configuration for multiple courses. This could be as multiple sites, but we do need a way to map their data assets. """ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + import sys import os @@ -67,6 +72,7 @@ MITX_FEATURES = { 'ENABLE_PSYCHOMETRICS': False, # real-time psychometrics (eg item response theory analysis in instructor dashboard) + 'ENABLE_DJANGO_ADMIN_SITE': False, # set true to enable django's admin site, even on prod (e.g. for course ops) 'ENABLE_SQL_TRACKING_LOGS': False, 'ENABLE_LMS_MIGRATION': False, 'ENABLE_MANUAL_GIT_RELOAD': False, @@ -105,6 +111,9 @@ MITX_FEATURES = { # Enable URL that shows information about the status of variuous services 'ENABLE_SERVICE_STATUS': False, + + # Toggle to indicate use of a custom theme + 'USE_CUSTOM_THEME': False } # Used for A/B testing @@ -162,12 +171,12 @@ MAKO_TEMPLATES['main'] = [PROJECT_ROOT / 'templates', # This is where Django Template lookup is defined. There are a few of these # still left lying around. -TEMPLATE_DIRS = ( +TEMPLATE_DIRS = [ PROJECT_ROOT / "templates", COMMON_ROOT / 'templates', COMMON_ROOT / 'lib' / 'capa' / 'capa' / 'templates', COMMON_ROOT / 'djangoapps' / 'pipeline_mako' / 'templates', -) +] TEMPLATE_CONTEXT_PROCESSORS = ( 'django.core.context_processors.request', @@ -709,3 +718,31 @@ MKTG_URL_LINK_MAP = { 'HONOR': 'honor', 'PRIVACY': 'privacy_edx', } + +############################### THEME ################################ +def enable_theme(theme_name): + """ + Enable the settings for a custom theme, whose files should be stored + in ENV_ROOT/themes/THEME_NAME (e.g., edx_all/themes/stanford). + + The THEME_NAME setting should be configured separately since it can't + be set here (this function closes too early). An idiom for doing this + is: + + THEME_NAME = "stanford" + enable_theme(THEME_NAME) + """ + MITX_FEATURES['USE_CUSTOM_THEME'] = True + + # Calculate the location of the theme's files + theme_root = ENV_ROOT / "themes" / theme_name + + # Include the theme's templates in the template search paths + TEMPLATE_DIRS.append(theme_root / 'templates') + MAKO_TEMPLATES['main'].append(theme_root / 'templates') + + # Namespace the theme's static files to 'themes/' to + # avoid collisions with default edX static files + STATICFILES_DIRS.append((u'themes/%s' % theme_name, + theme_root / 'static')) + diff --git a/lms/envs/content.py b/lms/envs/content.py index f699153895..f85ae0b9cd 100644 --- a/lms/envs/content.py +++ b/lms/envs/content.py @@ -2,6 +2,11 @@ These are debug machines used for content creators, so they're kind of a cross between dev machines and AWS machines. """ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .aws import * DEBUG = True diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 488110655e..9d7c0b3ac2 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -7,6 +7,11 @@ sessions. Assumes structure: /mitx # The location of this repo /log # Where we're going to write log files """ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .common import * from logsettings import get_logger_config diff --git a/lms/envs/dev_edx4edx.py b/lms/envs/dev_edx4edx.py index 2ebd24e68b..c90f369bc6 100644 --- a/lms/envs/dev_edx4edx.py +++ b/lms/envs/dev_edx4edx.py @@ -8,6 +8,10 @@ sessions. Assumes structure: /log # Where we're going to write log files """ +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + import socket if 'eecs1' in socket.gethostname(): diff --git a/lms/envs/dev_ike.py b/lms/envs/dev_ike.py index 639d186989..3f54b11d1e 100644 --- a/lms/envs/dev_ike.py +++ b/lms/envs/dev_ike.py @@ -7,6 +7,11 @@ sessions. Assumes structure: /mitx # The location of this repo /log # Where we're going to write log files """ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .common import * from logsettings import get_logger_config from .dev import * diff --git a/lms/envs/dev_int.py b/lms/envs/dev_int.py index 21c52c8abc..34921205a6 100644 --- a/lms/envs/dev_int.py +++ b/lms/envs/dev_int.py @@ -9,6 +9,11 @@ following domains to 127.0.0.1 in your /etc/hosts file: Note that OS X has a bug where using *.local domains is excruciatingly slow, so use *.dev domains instead for local testing. """ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .dev import * MITX_FEATURES['SUBDOMAIN_COURSE_LISTINGS'] = True diff --git a/lms/envs/dev_mongo.py b/lms/envs/dev_mongo.py index 6af0a429bb..dfbf473b45 100644 --- a/lms/envs/dev_mongo.py +++ b/lms/envs/dev_mongo.py @@ -1,6 +1,11 @@ """ This config file runs the dev environment, but with mongo as the datastore """ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .dev import * GITHUB_REPO_ROOT = ENV_ROOT / "data" diff --git a/lms/envs/dev_with_worker.py b/lms/envs/dev_with_worker.py index c5fc256ac9..078567c493 100644 --- a/lms/envs/dev_with_worker.py +++ b/lms/envs/dev_with_worker.py @@ -8,6 +8,10 @@ The worker can be executed using: django_admin.py celery worker """ +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from dev import * ################################# CELERY ###################################### diff --git a/lms/envs/devgroups/courses.py b/lms/envs/devgroups/courses.py index c44717c451..1a7ff58f08 100644 --- a/lms/envs/devgroups/courses.py +++ b/lms/envs/devgroups/courses.py @@ -1,3 +1,8 @@ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from ..dev import * CLASSES_TO_DBS = { diff --git a/lms/envs/devgroups/h_cs50.py b/lms/envs/devgroups/h_cs50.py index 9643c33d35..21c959f5ce 100644 --- a/lms/envs/devgroups/h_cs50.py +++ b/lms/envs/devgroups/h_cs50.py @@ -1,3 +1,8 @@ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .courses import * DATABASES = course_db_for('HarvardX/CS50x/2012') diff --git a/lms/envs/devgroups/m_6002.py b/lms/envs/devgroups/m_6002.py index 411e2bcc3c..d3c10fcd04 100644 --- a/lms/envs/devgroups/m_6002.py +++ b/lms/envs/devgroups/m_6002.py @@ -1,3 +1,8 @@ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .courses import * DATABASES = course_db_for('MITx/6.002x/2012_Fall') diff --git a/lms/envs/devgroups/portal.py b/lms/envs/devgroups/portal.py index 35808d56fa..8e4635cc66 100644 --- a/lms/envs/devgroups/portal.py +++ b/lms/envs/devgroups/portal.py @@ -2,6 +2,11 @@ Note that for this to work at all, you must have memcached running (or you won't get shared sessions) """ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from courses import * # Move this to a shared file later: diff --git a/lms/envs/devplus.py b/lms/envs/devplus.py index ea6590291c..bfd0788165 100644 --- a/lms/envs/devplus.py +++ b/lms/envs/devplus.py @@ -13,6 +13,11 @@ Dir structure: /log # Where we're going to write log files """ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .dev import * WIKI_ENABLED = True diff --git a/lms/envs/discussionsettings.py b/lms/envs/discussionsettings.py index f13680a7fe..1ac4c23af8 100644 --- a/lms/envs/discussionsettings.py +++ b/lms/envs/discussionsettings.py @@ -1 +1,5 @@ + +# We intentionally define variables that aren't used +# pylint: disable=W0614 + DISCUSSION_ALLOWED_UPLOAD_FILE_TYPES = ('.jpg', '.jpeg', '.gif', '.bmp', '.png', '.tiff') diff --git a/lms/envs/edx4edx_aws.py b/lms/envs/edx4edx_aws.py index b82048824f..247fa866bc 100644 --- a/lms/envs/edx4edx_aws.py +++ b/lms/envs/edx4edx_aws.py @@ -1,3 +1,7 @@ +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + # Settings for edx4edx production instance from .aws import * COURSE_NAME = "edx4edx" diff --git a/lms/envs/jasmine.py b/lms/envs/jasmine.py index ba4fcc5261..2c30bc7de7 100644 --- a/lms/envs/jasmine.py +++ b/lms/envs/jasmine.py @@ -2,6 +2,10 @@ This configuration is used for running jasmine tests """ +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .test import * from logsettings import get_logger_config diff --git a/lms/envs/static.py b/lms/envs/static.py index 23e735c747..260153e623 100644 --- a/lms/envs/static.py +++ b/lms/envs/static.py @@ -7,6 +7,11 @@ sessions. Assumes structure: /mitx # The location of this repo /log # Where we're going to write log files """ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .common import * from logsettings import get_logger_config diff --git a/lms/envs/test.py b/lms/envs/test.py index b8782ccd75..6691d50106 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -7,6 +7,11 @@ sessions. Assumes structure: /mitx # The location of this repo /log # Where we're going to write log files """ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .common import * import os from path import path diff --git a/lms/envs/test_ike.py b/lms/envs/test_ike.py index 907b7eeadf..46f7df211c 100644 --- a/lms/envs/test_ike.py +++ b/lms/envs/test_ike.py @@ -7,6 +7,11 @@ sessions. Assumes structure: /mitx # The location of this repo /log # Where we're going to write log files """ + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + from .common import * from logsettings import get_logger_config import os diff --git a/lms/lib/comment_client/utils.py b/lms/lib/comment_client/utils.py index 1ce03ed3c7..0ff06fced7 100644 --- a/lms/lib/comment_client/utils.py +++ b/lms/lib/comment_client/utils.py @@ -31,14 +31,9 @@ def merge_dict(dic1, dic2): def perform_request(method, url, data_or_params=None, *args, **kwargs): if data_or_params is None: data_or_params = {} - tags = [ - "{k}:{v}".format(k=k, v=v) - for (k, v) in data_or_params.items() + [("method", method), ("url", url)] - if k != 'api_key' - ] data_or_params['api_key'] = settings.API_KEY try: - with dog_stats_api.timer('comment_client.request.time', tags=tags): + with dog_stats_api.timer('comment_client.request.time'): if method in ['post', 'put', 'patch']: response = requests.request(method, url, data=data_or_params, timeout=5) else: @@ -55,6 +50,9 @@ def perform_request(method, url, data_or_params=None, *args, **kwargs): if 200 < response.status_code < 500: raise CommentClientError(response.text) + # Heroku returns a 503 when an application is in maintenance mode + elif response.status_code == 503: + raise CommentClientMaintenanceError(response.text) elif response.status_code == 500: raise CommentClientUnknownError(response.text) else: @@ -72,5 +70,9 @@ class CommentClientError(Exception): return repr(self.message) +class CommentClientMaintenanceError(CommentClientError): + pass + + class CommentClientUnknownError(CommentClientError): pass diff --git a/lms/static/sass/application.scss b/lms/static/sass/application.scss.mako similarity index 63% rename from lms/static/sass/application.scss rename to lms/static/sass/application.scss.mako index 6a1ef8743e..c310347b6f 100644 --- a/lms/static/sass/application.scss +++ b/lms/static/sass/application.scss.mako @@ -36,3 +36,16 @@ @import 'news'; @import 'shame'; + +## THEMING +## ------- +## Set up this file to import an edX theme library if the environment +## indicates that a theme should be used. The assumption is that the +## theme resides outside of this main edX repository, in a directory +## called themes//, with its base Sass file in +## themes//static/sass/_.scss. That one entry +## point can be used to @import in as many other things as needed. +% if env.get('THEME_NAME') is not None: + // import theme's Sass overrides + @import '${env.get('THEME_NAME')}' +% endif diff --git a/lms/templates/discussion/maintenance.html b/lms/templates/discussion/maintenance.html new file mode 100644 index 0000000000..a0f57cdfb3 --- /dev/null +++ b/lms/templates/discussion/maintenance.html @@ -0,0 +1,3 @@ +<%inherit file="../main.html" /> +

We're sorry

+

The forums are currently undergoing maintenance. We'll have them back up shortly!

diff --git a/lms/urls.py b/lms/urls.py index 3b14b41bd7..d1bee076fc 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -8,7 +8,7 @@ from . import one_time_startup import django.contrib.auth.views # Uncomment the next two lines to enable the admin: -if settings.DEBUG: +if settings.DEBUG or settings.MITX_FEATURES.get('ENABLE_DJANGO_ADMIN_SITE'): admin.autodiscover() urlpatterns = ('', # nopep8 @@ -330,7 +330,7 @@ if settings.COURSEWARE_ENABLED: if settings.ENABLE_JASMINE: urlpatterns += (url(r'^_jasmine/', include('django_jasmine.urls')),) -if settings.DEBUG: +if settings.DEBUG or settings.MITX_FEATURES.get('ENABLE_DJANGO_ADMIN_SITE'): ## Jasmine and admin urlpatterns += (url(r'^admin/', include(admin.site.urls)),) diff --git a/.pylintrc b/pylintrc similarity index 100% rename from .pylintrc rename to pylintrc diff --git a/rakefile b/rakefile index cef93e67eb..20101a14db 100644 --- a/rakefile +++ b/rakefile @@ -1,3 +1,4 @@ +require 'json' require 'rake/clean' require './rakefiles/helpers.rb' @@ -7,6 +8,13 @@ end # Build Constants REPO_ROOT = File.dirname(__FILE__) +ENV_ROOT = File.dirname(REPO_ROOT) REPORT_DIR = File.join(REPO_ROOT, "reports") +# Environment constants +SERVICE_VARIANT = ENV['SERVICE_VARIANT'] +CONFIG_PREFIX = SERVICE_VARIANT ? SERVICE_VARIANT + "." : "" +ENV_FILE = File.join(ENV_ROOT, CONFIG_PREFIX + "env.json") +ENV_TOKENS = File.exists?(ENV_FILE) ? JSON.parse(File.read(ENV_FILE)) : {} + task :default => [:test, :pep8, :pylint] diff --git a/rakefiles/assets.rake b/rakefiles/assets.rake index 0369968fdf..c0757b712b 100644 --- a/rakefiles/assets.rake +++ b/rakefiles/assets.rake @@ -1,3 +1,31 @@ +# Theming constants +THEME_NAME = ENV_TOKENS['THEME_NAME'] +USE_CUSTOM_THEME = !(THEME_NAME.nil? || THEME_NAME.empty?) +if USE_CUSTOM_THEME + THEME_ROOT = File.join(ENV_ROOT, "themes", THEME_NAME) + THEME_SASS = File.join(THEME_ROOT, "static", "sass") +end + +# Run the specified file through the Mako templating engine, providing +# the ENV_TOKENS to the templating context. +def preprocess_with_mako(filename) + # simple command-line invocation of Mako engine + mako = "from mako.template import Template;" + + "print Template(filename=\"#{filename}\")" + + # Total hack. It works because a Python dict literal has + # the same format as a JSON object. + ".render(env=#{ENV_TOKENS.to_json});" + + # strip off the .mako extension + output_filename = filename.chomp(File.extname(filename)) + + # just pipe from stdout into the new file, exiting on failure + File.open(output_filename, 'w') do |file| + file.write(`python -c '#{mako}'`) + exit_code = $?.to_i + abort "#{mako} failed with #{exit_code}" if exit_code.to_i != 0 + end +end def xmodule_cmd(watch=false, debug=false) xmodule_cmd = 'xmodule_assets common/static/xmodule' @@ -32,10 +60,17 @@ def coffee_cmd(watch=false, debug=false) end def sass_cmd(watch=false, debug=false) + sass_load_paths = ["./common/static/sass"] + sass_watch_paths = ["*/static"] + if USE_CUSTOM_THEME + sass_load_paths << THEME_SASS + sass_watch_paths << THEME_SASS + end + "sass #{debug ? '--debug-info' : '--style compressed'} " + - "--load-path ./common/static/sass " + + "--load-path #{sass_load_paths.join(' ')} " + "--require ./common/static/sass/bourbon/lib/bourbon.rb " + - "#{watch ? '--watch' : '--update'} */static" + "#{watch ? '--watch' : '--update'} #{sass_watch_paths.join(' ')}" end desc "Compile all assets" @@ -46,6 +81,13 @@ namespace :assets do desc "Compile all assets in debug mode" multitask :debug + desc "Preprocess all static assets that have the .mako extension" + task :preprocess do + # Run assets through the Mako templating engine. Right now we + # just hardcode the asset filenames. + preprocess_with_mako("lms/static/sass/application.scss.mako") + end + desc "Watch all assets for changes and automatically recompile" task :watch => 'assets:_watch' do puts "Press ENTER to terminate".red @@ -54,9 +96,9 @@ namespace :assets do {:xmodule => :install_python_prereqs, :coffee => :install_node_prereqs, - :sass => :install_ruby_prereqs}.each_pair do |asset_type, prereq_task| + :sass => [:install_ruby_prereqs, :preprocess]}.each_pair do |asset_type, prereq_tasks| desc "Compile all #{asset_type} assets" - task asset_type => prereq_task do + task asset_type => prereq_tasks do cmd = send(asset_type.to_s + "_cmd", watch=false, debug=false) if cmd.kind_of?(Array) cmd.each {|c| sh(c)} @@ -71,7 +113,7 @@ namespace :assets do namespace asset_type do desc "Compile all #{asset_type} assets in debug mode" - task :debug => prereq_task do + task :debug => prereq_tasks do cmd = send(asset_type.to_s + "_cmd", watch=false, debug=true) sh(cmd) end @@ -82,7 +124,7 @@ namespace :assets do $stdin.gets end - task :_watch => prereq_task do + task :_watch => prereq_tasks do cmd = send(asset_type.to_s + "_cmd", watch=true, debug=true) if cmd.kind_of?(Array) cmd.each {|c| background_process(c)} diff --git a/rakefiles/helpers.rb b/rakefiles/helpers.rb index 4c4d400b8a..f344aa2042 100644 --- a/rakefiles/helpers.rb +++ b/rakefiles/helpers.rb @@ -14,16 +14,31 @@ def report_dir_path(dir) return File.join(REPORT_DIR, dir.to_s) end -def when_changed(unchanged_message, *files) - Rake::Task[PREREQS_MD5_DIR].invoke - cache_file = File.join(PREREQS_MD5_DIR, files.join('-').gsub(/\W+/, '-')) + '.md5' +def compute_fingerprint(files, dirs) digest = Digest::MD5.new() + + # Digest the contents of all the files. Dir[*files].select{|file| File.file?(file)}.each do |file| digest.file(file) end - if !File.exists?(cache_file) or digest.hexdigest != File.read(cache_file) + + # Digest the names of the files in all the dirs. + dirs.each do |dir| + file_names = Dir.entries(dir).sort.join(" ") + digest.update(file_names) + end + + digest.hexdigest +end + +# Hash the contents of all the files, and the names of files in the dirs. +# Run the block if they've changed. +def when_changed(unchanged_message, files, dirs=[]) + Rake::Task[PREREQS_MD5_DIR].invoke + cache_file = File.join(PREREQS_MD5_DIR, files[0].gsub(/\W+/, '-').sub(/-+$/, '')) + '.md5' + if !File.exists?(cache_file) or compute_fingerprint(files, dirs) != File.read(cache_file) yield - File.write(cache_file, digest.hexdigest) + File.write(cache_file, compute_fingerprint(files, dirs)) elsif !unchanged_message.empty? puts unchanged_message end diff --git a/rakefiles/prereqs.rake b/rakefiles/prereqs.rake index ef4958e9d7..f453372065 100644 --- a/rakefiles/prereqs.rake +++ b/rakefiles/prereqs.rake @@ -1,6 +1,5 @@ require './rakefiles/helpers.rb' - PREREQS_MD5_DIR = ENV["PREREQ_CACHE_DIR"] || File.join(REPO_ROOT, '.prereqs_cache') CLOBBER.include(PREREQS_MD5_DIR) @@ -13,7 +12,7 @@ task :install_prereqs => [:install_node_prereqs, :install_ruby_prereqs, :install desc "Install all node prerequisites for the lms and cms" task :install_node_prereqs => "ws:migrate" do unchanged = 'Node requirements unchanged, nothing to install' - when_changed(unchanged, 'package.json') do + when_changed(unchanged, ['package.json']) do sh('npm install') end unless ENV['NO_PREREQ_INSTALL'] end @@ -21,20 +20,21 @@ end desc "Install all ruby prerequisites for the lms and cms" task :install_ruby_prereqs => "ws:migrate" do unchanged = 'Ruby requirements unchanged, nothing to install' - when_changed(unchanged, 'Gemfile') do + when_changed(unchanged, ['Gemfile']) do sh('bundle install') end unless ENV['NO_PREREQ_INSTALL'] end desc "Install all python prerequisites for the lms and cms" task :install_python_prereqs => "ws:migrate" do + site_packages_dir = `python -c 'import os; import distutils.sysconfig as dusc; print dusc.get_python_lib()'`.chomp unchanged = 'Python requirements unchanged, nothing to install' - when_changed(unchanged, 'requirements/**/*') do + when_changed(unchanged, ['requirements/**/*'], [site_packages_dir]) do ENV['PIP_DOWNLOAD_CACHE'] ||= '.pip_download_cache' sh('pip install --exists-action w -r requirements/edx/base.txt') sh('pip install --exists-action w -r requirements/edx/post.txt') - # Check for private-requirements.txt: used to install our libs as working dirs, - # or personal-use tools. + # requirements/private.txt is used to install our libs as + # working dirs, or for personal-use tools. if File.file?("requirements/private.txt") sh('pip install -r requirements/private.txt') end diff --git a/rakefiles/quality.rake b/rakefiles/quality.rake index 00ce627ac5..927f765eb5 100644 --- a/rakefiles/quality.rake +++ b/rakefiles/quality.rake @@ -1,3 +1,20 @@ +def run_pylint(system, report_dir, flags='') + apps = Dir["#{system}", "#{system}/djangoapps/*", "#{system}/lib/*"].map do |app| + File.basename(app) + end.select do |app| + app !=~ /.pyc$/ + end.map do |app| + if app =~ /.py$/ + app.gsub('.py', '') + else + app + end + end + + pythonpath_prefix = "PYTHONPATH=#{system}:#{system}/djangoapps:#{system}/lib:common/djangoapps:common/lib" + sh("#{pythonpath_prefix} pylint #{flags} -f parseable #{apps.join(' ')} | tee #{report_dir}/pylint.report") +end + [:lms, :cms, :common].each do |system| report_dir = report_dir_path(system) @@ -11,21 +28,18 @@ desc "Run pylint on all #{system} code" task "pylint_#{system}" => [report_dir, :install_python_prereqs] do - apps = Dir["#{system}/*.py", "#{system}/djangoapps/*", "#{system}/lib/*"].map do |app| - File.basename(app) - end.select do |app| - app !=~ /.pyc$/ - end.map do |app| - if app =~ /.py$/ - app.gsub('.py', '') - else - app - end - end - - pythonpath_prefix = "PYTHONPATH=#{system}:#{system}/djangoapps:#{system}/lib:common/djangoapps:common/lib" - sh("#{pythonpath_prefix} pylint --rcfile=.pylintrc -f parseable #{apps.join(' ')} | tee #{report_dir}/pylint.report") + run_pylint(system, report_dir) end + namespace "pylint_#{system}" do + desc "Run pylint checking for errors only, and aborting if there are any" + task :errors do + run_pylint(system, report_dir, '-E') + end + end + namespace :pylint do + task :errors => "pylint_#{system}:errors" + end + task :pylint => "pylint_#{system}" end \ No newline at end of file diff --git a/requirements/edx-sandbox/local.txt b/requirements/edx-sandbox/local.txt new file mode 100644 index 0000000000..ba24805057 --- /dev/null +++ b/requirements/edx-sandbox/local.txt @@ -0,0 +1,6 @@ +# Install these packages from the edx-platform working tree +# NOTE: if you change code in these packages, you MUST change the version +# number in its setup.py or the code WILL NOT be installed during deploy. +common/lib/calc +common/lib/chem +common/lib/sandbox-packages diff --git a/requirements/edx-sandbox/post.txt b/requirements/edx-sandbox/post.txt index f99e8a8c4b..218fdf307e 100644 --- a/requirements/edx-sandbox/post.txt +++ b/requirements/edx-sandbox/post.txt @@ -1,6 +1,3 @@ # Packages to install in the Python sandbox for secured execution. scipy==0.11.0 lxml==3.0.1 --e common/lib/calc --e common/lib/chem --e common/lib/sandbox-packages diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index ef0209ee03..3d8b95f8e2 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -74,6 +74,7 @@ lettuce==0.2.16 mock==0.8.0 nosexcover==1.0.7 pep8==1.4.5 +pylint==0.28 rednose==0.3 selenium==2.31.0 splinter==0.5.0 @@ -81,7 +82,3 @@ django_nose==1.1 django-jasmine==0.3.2 django_debug_toolbar django-debug-toolbar-mongo - -# Install pylint from a specific commit on trunk -# to get the fix for this issue: http://www.logilab.org/ticket/122793 -https://bitbucket.org/logilab/pylint/get/e828cb5.zip diff --git a/requirements/edx/github.txt b/requirements/edx/github.txt index d3f90d5abc..6b28d3edd9 100644 --- a/requirements/edx/github.txt +++ b/requirements/edx/github.txt @@ -8,5 +8,5 @@ -e git://github.com/eventbrite/zendesk.git@d53fe0e81b623f084e91776bcf6369f8b7b63879#egg=zendesk # Our libraries: --e git+https://github.com/edx/XBlock.git@483e0cb1#egg=XBlock +-e git+https://github.com/edx/XBlock.git@2144a25d#egg=XBlock -e git+https://github.com/edx/codejail.git@07494f1#egg=codejail