diff --git a/.gitignore b/.gitignore index 4fd90cfe03..72de96e0c4 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,4 @@ autodeploy.properties .ws_migrations_complete .vagrant/ logs +.testids/ diff --git a/.pep8 b/.pep8 index 25d0edbcb4..badb0219a0 100644 --- a/.pep8 +++ b/.pep8 @@ -1,2 +1,3 @@ [pep8] -ignore=E501 \ No newline at end of file +ignore=E501 +exclude=migrations \ No newline at end of file diff --git a/AUTHORS b/AUTHORS index 4b57d723d2..0391bd55f9 100644 --- a/AUTHORS +++ b/AUTHORS @@ -84,3 +84,5 @@ Mukul Goyal Robert Marks Yarko Tymciurak Miles Steele +Kevin Luo +Akshay Jagadeesh diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 749b9ef56e..89f084b3f4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,9 +5,15 @@ These are notable changes in edx-platform. This is a rolling list of changes, in roughly chronological order, most recent first. Add your entries at or near the top. Include a label indicating the component affected. +LMS: Added alphabetical sorting of forum categories and subcategories. +It is hidden behind a false defaulted course level flag. + Studio: Allow course authors to set their course image on the schedule and details page, with support for JPEG and PNG images. +LMS, Studio: Centralized startup code to manage.py and wsgi.py files. +Made studio runnable using wsgi. + Blades: Took videoalpha out of alpha, replacing the old video player Common: Allow instructors to input complicated expressions as answers to @@ -27,6 +33,9 @@ logic has been consolidated into the model -- you should use new class methods to `enroll()`, `unenroll()`, and to check `is_enrolled()`, instead of creating CourseEnrollment objects or querying them directly. +LMS: Added bulk email for course feature, with option to optout of individual +course emails. + Studio: Email will be sent to admin address when a user requests course creator privileges for Studio (edge only). diff --git a/cms/djangoapps/contentstore/features/advanced-settings.py b/cms/djangoapps/contentstore/features/advanced-settings.py index 18e179abdb..201ac49e52 100644 --- a/cms/djangoapps/contentstore/features/advanced-settings.py +++ b/cms/djangoapps/contentstore/features/advanced-settings.py @@ -2,7 +2,7 @@ #pylint: disable=W0621 from lettuce import world, step -from nose.tools import assert_false, assert_equal, assert_regexp_matches +from nose.tools import assert_false, assert_equal, assert_regexp_matches # pylint: disable=E0611 from common import type_in_codemirror, press_the_notification_button KEY_CSS = '.key input.policy-key' diff --git a/cms/djangoapps/contentstore/features/checklists.py b/cms/djangoapps/contentstore/features/checklists.py index e8dcd755a3..1c41eed4d3 100644 --- a/cms/djangoapps/contentstore/features/checklists.py +++ b/cms/djangoapps/contentstore/features/checklists.py @@ -2,7 +2,7 @@ #pylint: disable=W0621 from lettuce import world, step -from nose.tools import assert_true, assert_equal, assert_in +from nose.tools import assert_true, assert_equal, assert_in # pylint: disable=E0611 from terrain.steps import reload_the_page from selenium.common.exceptions import StaleElementReferenceException diff --git a/cms/djangoapps/contentstore/features/common.py b/cms/djangoapps/contentstore/features/common.py index 5d6fde47c8..a6f22db340 100644 --- a/cms/djangoapps/contentstore/features/common.py +++ b/cms/djangoapps/contentstore/features/common.py @@ -2,7 +2,7 @@ # pylint: disable=W0621 from lettuce import world, step -from nose.tools import assert_true +from nose.tools import assert_true # pylint: disable=E0611 from auth.authz import get_user_by_email, get_course_groupname_for_role from django.conf import settings @@ -265,9 +265,8 @@ def type_in_codemirror(index, text): def upload_file(filename): - file_css = '.upload-dialog input[type=file]' - upload = world.css_find(file_css).first path = os.path.join(TEST_ROOT, filename) - upload._element.send_keys(os.path.abspath(path)) + world.browser.execute_script("$('input.file-input').css('display', 'block')") + world.browser.attach_file('file', os.path.abspath(path)) button_css = '.upload-dialog .action-upload' world.css_click(button_css) diff --git a/cms/djangoapps/contentstore/features/component.py b/cms/djangoapps/contentstore/features/component.py index 15727dd992..d0c1fd59e7 100644 --- a/cms/djangoapps/contentstore/features/component.py +++ b/cms/djangoapps/contentstore/features/component.py @@ -2,7 +2,7 @@ #pylint: disable=W0621 from lettuce import world, step -from nose.tools import assert_true +from nose.tools import assert_true # pylint: disable=E0611 DATA_LOCATION = 'i4x://edx/templates' diff --git a/cms/djangoapps/contentstore/features/component_settings_editor_helpers.py b/cms/djangoapps/contentstore/features/component_settings_editor_helpers.py index 606e3dcee8..2971085081 100644 --- a/cms/djangoapps/contentstore/features/component_settings_editor_helpers.py +++ b/cms/djangoapps/contentstore/features/component_settings_editor_helpers.py @@ -2,7 +2,7 @@ #pylint: disable=C0111 from lettuce import world -from nose.tools import assert_equal +from nose.tools import assert_equal # pylint: disable=E0611 from terrain.steps import reload_the_page diff --git a/cms/djangoapps/contentstore/features/course-overview.py b/cms/djangoapps/contentstore/features/course-overview.py index 3fcb134f5b..289dbec308 100644 --- a/cms/djangoapps/contentstore/features/course-overview.py +++ b/cms/djangoapps/contentstore/features/course-overview.py @@ -3,7 +3,7 @@ from lettuce import world, step from common import * -from nose.tools import assert_true, assert_false, assert_equal +from nose.tools import assert_true, assert_false, assert_equal # pylint: disable=E0611 from logging import getLogger logger = getLogger(__name__) diff --git a/cms/djangoapps/contentstore/features/course-settings.py b/cms/djangoapps/contentstore/features/course-settings.py index 570c49a8c4..7004b9f99e 100644 --- a/cms/djangoapps/contentstore/features/course-settings.py +++ b/cms/djangoapps/contentstore/features/course-settings.py @@ -7,7 +7,7 @@ from selenium.webdriver.common.keys import Keys from common import type_in_codemirror, upload_file from django.conf import settings -from nose.tools import assert_true, assert_false, assert_equal +from nose.tools import assert_true, assert_false, assert_equal # pylint: disable=E0611 TEST_ROOT = settings.COMMON_TEST_DATA_ROOT @@ -168,15 +168,18 @@ def i_see_new_course_image(_step): img = images[0] expected_src = '/c4x/MITx/999/asset/image.jpg' # Don't worry about the domain in the URL - assert img['src'].endswith(expected_src) + try: + assert img['src'].endswith(expected_src) + except AssertionError as e: + e.args += ('Was looking for {}'.format(expected_src), 'Found {}'.format(img['src'])) + raise @step('the image URL should be present in the field') def image_url_present(_step): field_css = '#course-image-url' - field = world.css_find(field_css).first expected_value = '/c4x/MITx/999/asset/image.jpg' - assert field.value == expected_value + assert world.css_value(field_css) == expected_value ############### HELPER METHODS #################### diff --git a/cms/djangoapps/contentstore/features/course-team.py b/cms/djangoapps/contentstore/features/course-team.py index 8b31d325e5..85044dbbad 100644 --- a/cms/djangoapps/contentstore/features/course-team.py +++ b/cms/djangoapps/contentstore/features/course-team.py @@ -5,7 +5,7 @@ from lettuce import world, step from common import create_studio_user from django.contrib.auth.models import Group from auth.authz import get_course_groupname_for_role, get_user_by_email -from nose.tools import assert_true +from nose.tools import assert_true # pylint: disable=E0611 PASSWORD = 'test' EMAIL_EXTENSION = '@edx.org' diff --git a/cms/djangoapps/contentstore/features/course-updates.feature b/cms/djangoapps/contentstore/features/course-updates.feature index 41ee785db5..bc73479c5f 100644 --- a/cms/djangoapps/contentstore/features/course-updates.feature +++ b/cms/djangoapps/contentstore/features/course-updates.feature @@ -45,3 +45,25 @@ Feature: Course updates When I modify the handout to "
    Test
" Then I see the handout "Test" And I see a "saving" notification + + Scenario: Static links are rewritten when previewing a course update + Given I have opened a new course in Studio + And I go to the course updates page + When I add a new update with the text "" + # Can only do partial text matches because of the quotes with in quotes (and regexp step matching). + Then I should see the update "/c4x/MITx/999/asset/my_img.jpg" + And I change the update from "/static/my_img.jpg" to "" + Then I should see the update "/c4x/MITx/999/asset/modified.jpg" + And when I reload the page + Then I should see the update "/c4x/MITx/999/asset/modified.jpg" + + Scenario: Static links are rewritten when previewing handouts + Given I have opened a new course in Studio + And I go to the course updates page + When I modify the handout to "
" + # Can only do partial text matches because of the quotes with in quotes (and regexp step matching). + Then I see the handout "/c4x/MITx/999/asset/my_img.jpg" + And I change the handout from "/static/my_img.jpg" to "" + Then I see the handout "/c4x/MITx/999/asset/modified.jpg" + And when I reload the page + Then I see the handout "/c4x/MITx/999/asset/modified.jpg" diff --git a/cms/djangoapps/contentstore/features/course-updates.py b/cms/djangoapps/contentstore/features/course-updates.py index f431af9cf5..3278805a48 100644 --- a/cms/djangoapps/contentstore/features/course-updates.py +++ b/cms/djangoapps/contentstore/features/course-updates.py @@ -38,6 +38,16 @@ def modify_update(_step, text): change_text(text) +@step(u'I change the update from "([^"]*)" to "([^"]*)"$') +def change_existing_update(_step, before, after): + verify_text_in_editor_and_update('div.post-preview a.edit-button', before, after) + + +@step(u'I change the handout from "([^"]*)" to "([^"]*)"$') +def change_existing_handout(_step, before, after): + verify_text_in_editor_and_update('div.course-handouts a.edit-button', before, after) + + @step(u'I delete the update$') def click_button(_step): button_css = 'div.post-preview a.delete-button' @@ -80,3 +90,10 @@ def change_text(text): type_in_codemirror(0, text) save_css = 'a.save-button' world.css_click(save_css) + + +def verify_text_in_editor_and_update(button_css, before, after): + world.css_click(button_css) + text = world.css_find(".cm-string").html + assert before in text + change_text(after) diff --git a/cms/djangoapps/contentstore/features/problem-editor.py b/cms/djangoapps/contentstore/features/problem-editor.py index d891789e4a..5e4fe6364d 100644 --- a/cms/djangoapps/contentstore/features/problem-editor.py +++ b/cms/djangoapps/contentstore/features/problem-editor.py @@ -2,7 +2,7 @@ #pylint: disable=C0111 from lettuce import world, step -from nose.tools import assert_equal +from nose.tools import assert_equal # pylint: disable=E0611 from common import type_in_codemirror DISPLAY_NAME = "Display Name" diff --git a/cms/djangoapps/contentstore/features/section.py b/cms/djangoapps/contentstore/features/section.py index 3ca8e1676d..3fea8637c6 100644 --- a/cms/djangoapps/contentstore/features/section.py +++ b/cms/djangoapps/contentstore/features/section.py @@ -3,7 +3,7 @@ from lettuce import world, step from common import * -from nose.tools import assert_equal +from nose.tools import assert_equal # pylint: disable=E0611 ############### ACTIONS #################### diff --git a/cms/djangoapps/contentstore/features/static-pages.feature b/cms/djangoapps/contentstore/features/static-pages.feature index c1a8ec91fc..67652ea8f1 100644 --- a/cms/djangoapps/contentstore/features/static-pages.feature +++ b/cms/djangoapps/contentstore/features/static-pages.feature @@ -11,8 +11,9 @@ Feature: Static Pages Given I have opened a new course in Studio And I go to the static pages page And I add a new page - When I will confirm all alerts And I "delete" the "Empty" page + Then I am shown a prompt + When I confirm the prompt Then I should not see a "Empty" static page # Safari won't update the name properly diff --git a/cms/djangoapps/contentstore/features/subsection.py b/cms/djangoapps/contentstore/features/subsection.py index 60a325f550..6d9612d9bd 100644 --- a/cms/djangoapps/contentstore/features/subsection.py +++ b/cms/djangoapps/contentstore/features/subsection.py @@ -3,7 +3,7 @@ from lettuce import world, step from common import * -from nose.tools import assert_equal +from nose.tools import assert_equal # pylint: disable=E0611 ############### ACTIONS #################### diff --git a/cms/djangoapps/contentstore/features/video-editor.py b/cms/djangoapps/contentstore/features/video-editor.py index 312e2d545f..93d3be2ac0 100644 --- a/cms/djangoapps/contentstore/features/video-editor.py +++ b/cms/djangoapps/contentstore/features/video-editor.py @@ -29,7 +29,7 @@ def correct_video_settings(_step): ['Download Track', '', False], ['Download Video', '', False], ['End Time', '0', False], - ['HTML5 Subtitles', '', False], + ['HTML5 Timed Transcript', '', False], ['Show Captions', 'True', False], ['Start Time', '0', False], ['Video Sources', '', False], diff --git a/cms/djangoapps/contentstore/management/commands/check_course.py b/cms/djangoapps/contentstore/management/commands/check_course.py index 2f0b0b2a2c..13ac6af50c 100644 --- a/cms/djangoapps/contentstore/management/commands/check_course.py +++ b/cms/djangoapps/contentstore/management/commands/check_course.py @@ -3,11 +3,6 @@ from xmodule.modulestore.django import modulestore from xmodule.modulestore.xml_importer import check_module_metadata_editability from xmodule.course_module import CourseDescriptor -from request_cache.middleware import RequestCache - -from django.core.cache import get_cache - -CACHE = get_cache('mongo_metadata_inheritance') class Command(BaseCommand): help = '''Enumerates through the course and find common errors''' @@ -21,12 +16,6 @@ class Command(BaseCommand): loc = CourseDescriptor.id_to_location(loc_str) store = modulestore() - # setup a request cache so we don't throttle the DB with all the metadata inheritance requests - store.set_modulestore_configuration({ - 'metadata_inheritance_cache_subsystem': CACHE, - 'request_cache': RequestCache.get_request_cache() - }) - course = store.get_item(loc, depth=3) err_cnt = 0 diff --git a/cms/djangoapps/contentstore/management/commands/clone_course.py b/cms/djangoapps/contentstore/management/commands/clone_course.py index aa0e076f08..5ad0da09d8 100644 --- a/cms/djangoapps/contentstore/management/commands/clone_course.py +++ b/cms/djangoapps/contentstore/management/commands/clone_course.py @@ -9,14 +9,10 @@ from xmodule.course_module import CourseDescriptor from auth.authz import _copy_course_group + # # To run from command line: rake cms:clone SOURCE_LOC=MITx/111/Foo1 DEST_LOC=MITx/135/Foo3 # -from request_cache.middleware import RequestCache -from django.core.cache import get_cache - -CACHE = get_cache('mongo_metadata_inheritance') - class Command(BaseCommand): """Clone a MongoDB-backed course to another location""" help = 'Clone a MongoDB backed course to another location' @@ -32,11 +28,6 @@ class Command(BaseCommand): mstore = modulestore('direct') cstore = contentstore() - mstore.set_modulestore_configuration({ - 'metadata_inheritance_cache_subsystem': CACHE, - 'request_cache': RequestCache.get_request_cache() - }) - org, course_num, run = dest_course_id.split("/") mstore.ignore_write_events_on_courses.append('{0}/{1}'.format(org, course_num)) diff --git a/cms/djangoapps/contentstore/management/commands/delete_course.py b/cms/djangoapps/contentstore/management/commands/delete_course.py index b0901ccfc9..50f9b82e80 100644 --- a/cms/djangoapps/contentstore/management/commands/delete_course.py +++ b/cms/djangoapps/contentstore/management/commands/delete_course.py @@ -9,14 +9,11 @@ from xmodule.course_module import CourseDescriptor from .prompt import query_yes_no from auth.authz import _delete_course_group -from request_cache.middleware import RequestCache -from django.core.cache import get_cache + # # To run from command line: rake cms:delete_course LOC=MITx/111/Foo1 # - -CACHE = get_cache('mongo_metadata_inheritance') class Command(BaseCommand): help = '''Delete a MongoDB backed course''' @@ -36,11 +33,6 @@ class Command(BaseCommand): ms = modulestore('direct') cs = contentstore() - ms.set_modulestore_configuration({ - 'metadata_inheritance_cache_subsystem': CACHE, - 'request_cache': RequestCache.get_request_cache() - }) - org, course_num, run = course_id.split("/") ms.ignore_write_events_on_courses.append('{0}/{1}'.format(org, course_num)) diff --git a/cms/djangoapps/contentstore/tests/modulestore_config.py b/cms/djangoapps/contentstore/tests/modulestore_config.py new file mode 100644 index 0000000000..234fa66f9f --- /dev/null +++ b/cms/djangoapps/contentstore/tests/modulestore_config.py @@ -0,0 +1,8 @@ +""" +Define test configuration for modulestores. +""" + +from xmodule.modulestore.tests.django_utils import studio_store_config +from django.conf import settings + +TEST_MODULESTORE = studio_store_config(settings.TEST_ROOT / "data") diff --git a/cms/djangoapps/contentstore/tests/test_assets.py b/cms/djangoapps/contentstore/tests/test_assets.py index b627237729..2f158cfda6 100644 --- a/cms/djangoapps/contentstore/tests/test_assets.py +++ b/cms/djangoapps/contentstore/tests/test_assets.py @@ -60,11 +60,11 @@ class UploadTestCase(CourseTestCase): f = BytesIO("sample content") f.name = "sample.txt" resp = self.client.post(self.url, {"name": "my-name", "file": f}) - self.assert2XX(resp.status_code) + self.assertEquals(resp.status_code, 200) def test_no_file(self): resp = self.client.post(self.url, {"name": "file.txt"}) - self.assert4XX(resp.status_code) + self.assertEquals(resp.status_code, 400) def test_get(self): resp = self.client.get(self.url) diff --git a/cms/djangoapps/contentstore/tests/test_contentstore.py b/cms/djangoapps/contentstore/tests/test_contentstore.py index f03ee3b81a..696b60fbe5 100644 --- a/cms/djangoapps/contentstore/tests/test_contentstore.py +++ b/cms/djangoapps/contentstore/tests/test_contentstore.py @@ -3,6 +3,9 @@ import json import shutil import mock + +from textwrap import dedent + from django.test.client import Client from django.test.utils import override_settings from django.conf import settings @@ -22,6 +25,7 @@ from contentstore.tests.utils import parse_json from auth.authz import add_user_to_creator_group from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from contentstore.tests.modulestore_config import TEST_MODULESTORE from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory from xmodule.modulestore import Location, mongo @@ -65,7 +69,7 @@ class MongoCollectionFindWrapper(object): return self.original(query, *args, **kwargs) -@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE) +@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE) class ContentStoreToyCourseTest(ModuleStoreTestCase): """ Tests that rely on the toy courses. @@ -312,7 +316,14 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): handouts = module_store.get_item(Location(['i4x', 'edX', 'toy', 'html', 'toyhtml', None])) self.assertIn('/static/', handouts.data) - def test_import_textbook_as_content_element(self): + @mock.patch('xmodule.course_module.requests.get') + def test_import_textbook_as_content_element(self, mock_get): + mock_get.return_value.text = dedent(""" + + + + """).strip() + module_store = modulestore('direct') import_from_xml(module_store, 'common/test/data/', ['toy']) @@ -845,7 +856,14 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): filesystem = OSFS(root_dir / ('test_export/' + dirname)) self.assertTrue(filesystem.exists(item.location.name + filename_suffix)) - def test_export_course(self): + @mock.patch('xmodule.course_module.requests.get') + def test_export_course(self, mock_get): + mock_get.return_value.text = dedent(""" + + + + """).strip() + module_store = modulestore('direct') draft_store = modulestore('draft') content_store = contentstore() @@ -1122,12 +1140,15 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): wrapper = MongoCollectionFindWrapper(module_store.collection.find) module_store.collection.find = wrapper.find + print module_store.metadata_inheritance_cache_subsystem + print module_store.request_cache course = module_store.get_item(location, depth=2) # make sure we haven't done too many round trips to DB - # note we say 4 round trips here for 1) the course, 2 & 3) for the chapters and sequentials, and - # 4) because of the RT due to calculating the inherited metadata - self.assertEqual(wrapper.counter, 4) + # note we say 3 round trips here for 1) the course, and 2 & 3) for the chapters and sequentials + # Because we're querying from the top of the tree, we cache information needed for inheritance, + # so we don't need to make an extra query to compute it. + self.assertEqual(wrapper.counter, 3) # make sure we pre-fetched a known sequential which should be at depth=2 self.assertTrue(Location(['i4x', 'edX', 'toy', 'sequential', @@ -1163,7 +1184,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): export_to_xml(module_store, content_store, location, root_dir, 'test_export') -@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE) +@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE) class ContentStoreTest(ModuleStoreTestCase): """ Tests for the CMS ContentStore application. @@ -1408,7 +1429,7 @@ class ContentStoreTest(ModuleStoreTestCase): 'course': loc.course, 'name': loc.name})) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) self.assertContains(resp, 'Chapter 2') # go to various pages @@ -1418,92 +1439,92 @@ class ContentStoreTest(ModuleStoreTestCase): kwargs={'org': loc.org, 'course': loc.course, 'name': loc.name})) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) # export page resp = self.client.get(reverse('export_course', kwargs={'org': loc.org, 'course': loc.course, 'name': loc.name})) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) # manage users resp = self.client.get(reverse('manage_users', kwargs={'org': loc.org, 'course': loc.course, 'name': loc.name})) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) # course info resp = self.client.get(reverse('course_info', kwargs={'org': loc.org, 'course': loc.course, 'name': loc.name})) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) # settings_details resp = self.client.get(reverse('settings_details', kwargs={'org': loc.org, 'course': loc.course, 'name': loc.name})) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) # settings_details resp = self.client.get(reverse('settings_grading', kwargs={'org': loc.org, 'course': loc.course, 'name': loc.name})) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) # static_pages resp = self.client.get(reverse('static_pages', kwargs={'org': loc.org, 'course': loc.course, 'coursename': loc.name})) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) # static_pages resp = self.client.get(reverse('asset_index', kwargs={'org': loc.org, 'course': loc.course, 'name': loc.name})) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) # go look at a subsection page subsection_location = loc.replace(category='sequential', name='test_sequence') resp = self.client.get(reverse('edit_subsection', kwargs={'location': subsection_location.url()})) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) # go look at the Edit page unit_location = loc.replace(category='vertical', name='test_vertical') resp = self.client.get(reverse('edit_unit', kwargs={'location': unit_location.url()})) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) # delete a component del_loc = loc.replace(category='html', name='test_html') resp = self.client.post(reverse('delete_item'), json.dumps({'id': del_loc.url()}), "application/json") - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 204) # delete a unit del_loc = loc.replace(category='vertical', name='test_vertical') resp = self.client.post(reverse('delete_item'), json.dumps({'id': del_loc.url()}), "application/json") - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 204) # delete a unit del_loc = loc.replace(category='sequential', name='test_sequence') resp = self.client.post(reverse('delete_item'), json.dumps({'id': del_loc.url()}), "application/json") - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 204) # delete a chapter del_loc = loc.replace(category='chapter', name='chapter_2') resp = self.client.post(reverse('delete_item'), json.dumps({'id': del_loc.url()}), "application/json") - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 204) def test_import_into_new_course_id(self): module_store = modulestore('direct') @@ -1690,6 +1711,7 @@ class ContentStoreTest(ModuleStoreTestCase): content_store.find(location) +@override_settings(MODULESTORE=TEST_MODULESTORE) class MetadataSaveTestCase(ModuleStoreTestCase): """Test that metadata is correctly cached and decached.""" diff --git a/cms/djangoapps/contentstore/tests/test_course_settings.py b/cms/djangoapps/contentstore/tests/test_course_settings.py index dbdf8b3f6e..524dde07e5 100644 --- a/cms/djangoapps/contentstore/tests/test_course_settings.py +++ b/cms/djangoapps/contentstore/tests/test_course_settings.py @@ -439,12 +439,12 @@ class CourseGraderUpdatesTest(CourseTestCase): def test_get(self): resp = self.client.get(self.url) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) obj = json.loads(resp.content) def test_delete(self): resp = self.client.delete(self.url) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 204) def test_post(self): grader = { @@ -455,5 +455,5 @@ class CourseGraderUpdatesTest(CourseTestCase): "weight": 17.3, } resp = self.client.post(self.url, grader) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) obj = json.loads(resp.content) diff --git a/cms/djangoapps/contentstore/tests/test_i18n.py b/cms/djangoapps/contentstore/tests/test_i18n.py index e6baf57213..9e7a2df8b2 100644 --- a/cms/djangoapps/contentstore/tests/test_i18n.py +++ b/cms/djangoapps/contentstore/tests/test_i18n.py @@ -3,10 +3,13 @@ from unittest import skip from django.core.urlresolvers import reverse from django.contrib.auth.models import User from django.test.client import Client +from django.test.utils import override_settings from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from contentstore.tests.modulestore_config import TEST_MODULESTORE +@override_settings(MODULESTORE=TEST_MODULESTORE) class InternationalizationTest(ModuleStoreTestCase): """ Tests to validate Internationalization. diff --git a/cms/djangoapps/contentstore/tests/test_import_export.py b/cms/djangoapps/contentstore/tests/test_import_export.py new file mode 100644 index 0000000000..05f2b0b7b9 --- /dev/null +++ b/cms/djangoapps/contentstore/tests/test_import_export.py @@ -0,0 +1,94 @@ +""" +Unit tests for course import and export +""" +import os +import shutil +import tarfile +import tempfile +import copy +from uuid import uuid4 +from pymongo import MongoClient + +from .utils import CourseTestCase +from django.core.urlresolvers import reverse +from django.test.utils import override_settings +from django.conf import settings + +from xmodule.contentstore.django import _CONTENTSTORE + +TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) +TEST_DATA_CONTENTSTORE['OPTIONS']['db'] = 'test_xcontent_%s' % uuid4().hex + +@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE) +class ImportTestCase(CourseTestCase): + """ + Unit tests for importing a course + """ + + def setUp(self): + super(ImportTestCase, self).setUp() + self.url = reverse("import_course", kwargs={ + 'org': self.course.location.org, + 'course': self.course.location.course, + 'name': self.course.location.name, + }) + self.content_dir = tempfile.mkdtemp() + + def touch(name): + """ Equivalent to shell's 'touch'""" + with file(name, 'a'): + os.utime(name, None) + + # Create tar test files ----------------------------------------------- + # OK course: + good_dir = tempfile.mkdtemp(dir=self.content_dir) + os.makedirs(os.path.join(good_dir, "course")) + with open(os.path.join(good_dir, "course.xml"), "w+") as f: + f.write('') + + with open(os.path.join(good_dir, "course", "2013_Spring.xml"), "w+") as f: + f.write('') + + self.good_tar = os.path.join(self.content_dir, "good.tar.gz") + with tarfile.open(self.good_tar, "w:gz") as gtar: + gtar.add(good_dir) + + # Bad course (no 'course.xml' file): + bad_dir = tempfile.mkdtemp(dir=self.content_dir) + touch(os.path.join(bad_dir, "bad.xml")) + self.bad_tar = os.path.join(self.content_dir, "bad.tar.gz") + with tarfile.open(self.bad_tar, "w:gz") as btar: + btar.add(bad_dir) + + def tearDown(self): + shutil.rmtree(self.content_dir) + MongoClient().drop_database(TEST_DATA_CONTENTSTORE['OPTIONS']['db']) + _CONTENTSTORE.clear() + + def test_no_coursexml(self): + """ + Check that the response for a tar.gz import without a course.xml is + correct. + """ + with open(self.bad_tar) as btar: + resp = self.client.post( + self.url, + { + "name": self.bad_tar, + "course-data": [btar] + }) + self.assertEquals(resp.status_code, 415) + + def test_with_coursexml(self): + """ + Check that the response for a tar.gz import with a course.xml is + correct. + """ + with open(self.good_tar) as gtar: + resp = self.client.post( + self.url, + { + "name": self.good_tar, + "course-data": [gtar] + }) + self.assertEquals(resp.status_code, 200) diff --git a/cms/djangoapps/contentstore/tests/test_import_nostatic.py b/cms/djangoapps/contentstore/tests/test_import_nostatic.py index aad6ffbfe4..f0f65c9b07 100644 --- a/cms/djangoapps/contentstore/tests/test_import_nostatic.py +++ b/cms/djangoapps/contentstore/tests/test_import_nostatic.py @@ -12,24 +12,26 @@ import copy from django.contrib.auth.models import User from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from contentstore.tests.modulestore_config import TEST_MODULESTORE from xmodule.modulestore import Location from xmodule.modulestore.django import modulestore from xmodule.contentstore.django import contentstore from xmodule.modulestore.xml_importer import import_from_xml from xmodule.contentstore.content import StaticContent +from xmodule.contentstore.django import _CONTENTSTORE from xmodule.course_module import CourseDescriptor from xmodule.exceptions import NotFoundError from uuid import uuid4 - +from pymongo import MongoClient TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) TEST_DATA_CONTENTSTORE['OPTIONS']['db'] = 'test_xcontent_%s' % uuid4().hex -@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE) +@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE) class ContentStoreImportNoStaticTest(ModuleStoreTestCase): """ Tests that rely on the toy and test_import_course courses. @@ -58,6 +60,10 @@ class ContentStoreImportNoStaticTest(ModuleStoreTestCase): self.client = Client() self.client.login(username=uname, password=password) + def tearDown(self): + MongoClient().drop_database(TEST_DATA_CONTENTSTORE['OPTIONS']['db']) + _CONTENTSTORE.clear() + def load_test_import_course(self): ''' Load the standard course used to test imports (for do_import_static=False behavior). @@ -121,3 +127,9 @@ class ContentStoreImportNoStaticTest(ModuleStoreTestCase): handouts = module_store.get_item(Location(['i4x', 'edX', 'toy', 'html', 'toyhtml', None])) self.assertIn('/static/', handouts.data) + + def test_tab_name_imports_correctly(self): + module_store, content_store, course, course_location = self.load_test_import_course() + print "course tabs = {0}".format(course.tabs) + self.assertEqual(course.tabs[2]['name'],'Syllabus') + diff --git a/cms/djangoapps/contentstore/tests/test_item.py b/cms/djangoapps/contentstore/tests/test_item.py index 260444a8f7..e5ff992cb8 100644 --- a/cms/djangoapps/contentstore/tests/test_item.py +++ b/cms/djangoapps/contentstore/tests/test_item.py @@ -34,7 +34,7 @@ class DeleteItem(CourseTestCase): resp.content, "application/json" ) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 204) class TestCreateItem(CourseTestCase): diff --git a/cms/djangoapps/contentstore/tests/test_textbooks.py b/cms/djangoapps/contentstore/tests/test_textbooks.py index a21a1b1023..950d0f780e 100644 --- a/cms/djangoapps/contentstore/tests/test_textbooks.py +++ b/cms/djangoapps/contentstore/tests/test_textbooks.py @@ -23,7 +23,7 @@ class TextbookIndexTestCase(CourseTestCase): def test_view_index(self): "Basic check that the textbook index page responds correctly" resp = self.client.get(self.url) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) # we don't have resp.context right now, # due to bugs in our testing harness :( if resp.context: @@ -36,7 +36,7 @@ class TextbookIndexTestCase(CourseTestCase): HTTP_ACCEPT="application/json", HTTP_X_REQUESTED_WITH='XMLHttpRequest' ) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) obj = json.loads(resp.content) self.assertEqual(self.course.pdf_textbooks, obj) @@ -73,7 +73,7 @@ class TextbookIndexTestCase(CourseTestCase): HTTP_ACCEPT="application/json", HTTP_X_REQUESTED_WITH='XMLHttpRequest' ) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) obj = json.loads(resp.content) self.assertEqual(content, obj) @@ -90,7 +90,7 @@ class TextbookIndexTestCase(CourseTestCase): HTTP_ACCEPT="application/json", HTTP_X_REQUESTED_WITH='XMLHttpRequest' ) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) # reload course store = get_modulestore(self.course.location) @@ -111,7 +111,7 @@ class TextbookIndexTestCase(CourseTestCase): HTTP_ACCEPT="application/json", HTTP_X_REQUESTED_WITH='XMLHttpRequest' ) - self.assert4XX(resp.status_code) + self.assertEqual(resp.status_code, 400) obj = json.loads(resp.content) self.assertIn("error", obj) @@ -184,7 +184,7 @@ class TextbookCreateTestCase(CourseTestCase): HTTP_ACCEPT="application/json", HTTP_X_REQUESTED_WITH="XMLHttpRequest", ) - self.assert4XX(resp.status_code) + self.assertEqual(resp.status_code, 400) self.assertNotIn("Location", resp) @@ -238,14 +238,14 @@ class TextbookByIdTestCase(CourseTestCase): def test_get_1(self): "Get the first textbook" resp = self.client.get(self.url1) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) compare = json.loads(resp.content) self.assertEqual(compare, self.textbook1) def test_get_2(self): "Get the second textbook" resp = self.client.get(self.url2) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) compare = json.loads(resp.content) self.assertEqual(compare, self.textbook2) @@ -257,7 +257,7 @@ class TextbookByIdTestCase(CourseTestCase): def test_delete(self): "Delete a textbook by ID" resp = self.client.delete(self.url1) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 204) course = self.store.get_item(self.course.location) self.assertEqual(course.pdf_textbooks, [self.textbook2]) @@ -288,7 +288,7 @@ class TextbookByIdTestCase(CourseTestCase): ) self.assertEqual(resp.status_code, 201) resp2 = self.client.get(url) - self.assert2XX(resp2.status_code) + self.assertEqual(resp2.status_code, 200) compare = json.loads(resp2.content) self.assertEqual(compare, textbook) course = self.store.get_item(self.course.location) @@ -311,7 +311,7 @@ class TextbookByIdTestCase(CourseTestCase): ) self.assertEqual(resp.status_code, 201) resp2 = self.client.get(self.url2) - self.assert2XX(resp2.status_code) + self.assertEqual(resp2.status_code, 200) compare = json.loads(resp2.content) self.assertEqual(compare, replacement) course = self.store.get_item(self.course.location) diff --git a/cms/djangoapps/contentstore/tests/test_users.py b/cms/djangoapps/contentstore/tests/test_users.py index cbb8aa8b01..80b2364c43 100644 --- a/cms/djangoapps/contentstore/tests/test_users.py +++ b/cms/djangoapps/contentstore/tests/test_users.py @@ -72,13 +72,13 @@ class UsersTestCase(CourseTestCase): def test_detail_inactive(self): resp = self.client.get(self.inactive_detail_url) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 200) result = json.loads(resp.content) self.assertFalse(result["active"]) def test_detail_invalid(self): resp = self.client.get(self.invalid_detail_url) - self.assert4XX(resp.status_code) + self.assertEqual(resp.status_code, 404) result = json.loads(resp.content) self.assertIn("error", result) @@ -87,7 +87,7 @@ class UsersTestCase(CourseTestCase): self.detail_url, data={"role": None}, ) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 204) # reload user from DB ext_user = User.objects.get(email=self.ext_user.email) groups = [g.name for g in ext_user.groups.all()] @@ -103,7 +103,7 @@ class UsersTestCase(CourseTestCase): content_type="application/json", HTTP_ACCEPT="application/json", ) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 204) # reload user from DB ext_user = User.objects.get(email=self.ext_user.email) groups = [g.name for g in ext_user.groups.all()] @@ -122,7 +122,7 @@ class UsersTestCase(CourseTestCase): content_type="application/json", HTTP_ACCEPT="application/json", ) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 204) # reload user from DB ext_user = User.objects.get(email=self.ext_user.email) groups = [g.name for g in ext_user.groups.all()] @@ -142,7 +142,7 @@ class UsersTestCase(CourseTestCase): content_type="application/json", HTTP_ACCEPT="application/json", ) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 204) # reload user from DB ext_user = User.objects.get(email=self.ext_user.email) groups = [g.name for g in ext_user.groups.all()] @@ -157,7 +157,7 @@ class UsersTestCase(CourseTestCase): content_type="application/json", HTTP_ACCEPT="application/json", ) - self.assert4XX(resp.status_code) + self.assertEqual(resp.status_code, 400) result = json.loads(resp.content) self.assertIn("error", result) self.assert_not_enrolled() @@ -169,7 +169,7 @@ class UsersTestCase(CourseTestCase): content_type="application/json", HTTP_ACCEPT="application/json", ) - self.assert4XX(resp.status_code) + self.assertEqual(resp.status_code, 400) result = json.loads(resp.content) self.assertIn("error", result) self.assert_not_enrolled() @@ -180,7 +180,7 @@ class UsersTestCase(CourseTestCase): data={"role": "staff"}, HTTP_ACCEPT="application/json", ) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 204) # reload user from DB ext_user = User.objects.get(email=self.ext_user.email) groups = [g.name for g in ext_user.groups.all()] @@ -197,7 +197,7 @@ class UsersTestCase(CourseTestCase): self.detail_url, HTTP_ACCEPT="application/json", ) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 204) # reload user from DB ext_user = User.objects.get(email=self.ext_user.email) groups = [g.name for g in ext_user.groups.all()] @@ -214,7 +214,7 @@ class UsersTestCase(CourseTestCase): self.detail_url, HTTP_ACCEPT="application/json", ) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 204) # reload user from DB ext_user = User.objects.get(email=self.ext_user.email) groups = [g.name for g in ext_user.groups.all()] @@ -273,7 +273,7 @@ class UsersTestCase(CourseTestCase): data={"role": "instructor"}, HTTP_ACCEPT="application/json", ) - self.assert4XX(resp.status_code) + self.assertEqual(resp.status_code, 400) result = json.loads(resp.content) self.assertIn("error", result) @@ -288,7 +288,7 @@ class UsersTestCase(CourseTestCase): data={"role": "instructor"}, HTTP_ACCEPT="application/json", ) - self.assert4XX(resp.status_code) + self.assertEqual(resp.status_code, 400) result = json.loads(resp.content) self.assertIn("error", result) @@ -306,7 +306,7 @@ class UsersTestCase(CourseTestCase): }) resp = self.client.delete(self_url) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 204) # reload user from DB user = User.objects.get(email=self.user.email) groups = [g.name for g in user.groups.all()] @@ -321,7 +321,7 @@ class UsersTestCase(CourseTestCase): self.ext_user.save() resp = self.client.delete(self.detail_url) - self.assert4XX(resp.status_code) + self.assertEqual(resp.status_code, 400) result = json.loads(resp.content) self.assertIn("error", result) # reload user from DB @@ -347,7 +347,7 @@ class UsersTestCase(CourseTestCase): self.detail_url, HTTP_ACCEPT="application/json", ) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 204) self.assert_enrolled() def test_staff_to_instructor_still_enrolled(self): @@ -366,7 +366,7 @@ class UsersTestCase(CourseTestCase): content_type="application/json", HTTP_ACCEPT="application/json", ) - self.assert2XX(resp.status_code) + self.assertEqual(resp.status_code, 204) self.assert_enrolled() def assert_not_enrolled(self): diff --git a/cms/djangoapps/contentstore/tests/tests.py b/cms/djangoapps/contentstore/tests/tests.py index 0cbc82cbf1..eddf5ab25a 100644 --- a/cms/djangoapps/contentstore/tests/tests.py +++ b/cms/djangoapps/contentstore/tests/tests.py @@ -1,15 +1,18 @@ from django.test.client import Client +from django.test.utils import override_settings from django.core.cache import cache from django.core.urlresolvers import reverse -from .utils import parse_json, user, registration +from contentstore.tests.utils import parse_json, user, registration from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from contentstore.tests.test_course_settings import CourseTestCase from xmodule.modulestore.tests.factories import CourseFactory +from contentstore.tests.modulestore_config import TEST_MODULESTORE import datetime from pytz import UTC +@override_settings(MODULESTORE=TEST_MODULESTORE) class ContentStoreTestCase(ModuleStoreTestCase): def _login(self, email, password): """ diff --git a/cms/djangoapps/contentstore/tests/utils.py b/cms/djangoapps/contentstore/tests/utils.py index a3f211a703..8b3f4cf4b1 100644 --- a/cms/djangoapps/contentstore/tests/utils.py +++ b/cms/djangoapps/contentstore/tests/utils.py @@ -7,9 +7,11 @@ import json from student.models import Registration from django.contrib.auth.models import User from django.test.client import Client +from django.test.utils import override_settings from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory +from contentstore.tests.modulestore_config import TEST_MODULESTORE def parse_json(response): @@ -27,6 +29,7 @@ def registration(email): return Registration.objects.get(user__email=email) +@override_settings(MODULESTORE=TEST_MODULESTORE) class CourseTestCase(ModuleStoreTestCase): def setUp(self): """ diff --git a/cms/djangoapps/contentstore/views/__init__.py b/cms/djangoapps/contentstore/views/__init__.py index 197c54ff36..10f6fb79a7 100644 --- a/cms/djangoapps/contentstore/views/__init__.py +++ b/cms/djangoapps/contentstore/views/__init__.py @@ -10,6 +10,7 @@ from .component import * from .course import * from .error import * from .item import * +from .import_export import * from .preview import * from .public import * from .user import * diff --git a/cms/djangoapps/contentstore/views/assets.py b/cms/djangoapps/contentstore/views/assets.py index 74cb94a354..4743622fa8 100644 --- a/cms/djangoapps/contentstore/views/assets.py +++ b/cms/djangoapps/contentstore/views/assets.py @@ -4,6 +4,7 @@ import os import tarfile import shutil import cgi +import re from functools import partial from tempfile import mkdtemp from path import path @@ -35,9 +36,7 @@ from .access import get_location_and_verify_access from util.json_request import JsonResponse -__all__ = ['asset_index', 'upload_asset', 'import_course', - 'generate_export_course', 'export_course'] - +__all__ = ['asset_index', 'upload_asset'] def assets_to_json_dict(assets): """ @@ -167,7 +166,7 @@ def upload_asset(request, org, course, coursename): sc_partial = partial(StaticContent, content_loc, filename, mime_type) if chunked: content = sc_partial(upload_file.chunks()) - temp_filepath = upload_file.temporary_file_path() + tempfile_path = upload_file.temporary_file_path() else: content = sc_partial(upload_file.read()) tempfile_path = None @@ -260,179 +259,3 @@ def remove_asset(request, org, course, name): return HttpResponse() -@ensure_csrf_cookie -@require_http_methods(("GET", "POST", "PUT")) -@login_required -def import_course(request, org, course, name): - """ - This method will handle a POST request to upload and import a .tar.gz file into a specified course - """ - location = get_location_and_verify_access(request, org, course, name) - - if request.method in ('POST', 'PUT'): - filename = request.FILES['course-data'].name - - if not filename.endswith('.tar.gz'): - return HttpResponse(json.dumps({'ErrMsg': 'We only support uploading a .tar.gz file.'})) - - data_root = path(settings.GITHUB_REPO_ROOT) - - course_subdir = "{0}-{1}-{2}".format(org, course, name) - course_dir = data_root / course_subdir - if not course_dir.isdir(): - os.mkdir(course_dir) - - temp_filepath = course_dir / filename - - logging.debug('importing course to {0}'.format(temp_filepath)) - - # stream out the uploaded files in chunks to disk - temp_file = open(temp_filepath, 'wb+') - for chunk in request.FILES['course-data'].chunks(): - temp_file.write(chunk) - temp_file.close() - - tar_file = tarfile.open(temp_filepath) - tar_file.extractall(course_dir + '/') - - # find the 'course.xml' file - dirpath = None - for dirpath, _dirnames, filenames in os.walk(course_dir): - for filename in filenames: - if filename == 'course.xml': - break - if filename == 'course.xml': - break - - if filename != 'course.xml': - return HttpResponse(json.dumps({'ErrMsg': 'Could not find the course.xml file in the package.'})) - - logging.debug('found course.xml at {0}'.format(dirpath)) - - if dirpath != course_dir: - for fname in os.listdir(dirpath): - shutil.move(dirpath / fname, course_dir) - - _module_store, course_items = import_from_xml(modulestore('direct'), settings.GITHUB_REPO_ROOT, - [course_subdir], load_error_modules=False, - static_content_store=contentstore(), - target_location_namespace=location, - draft_store=modulestore()) - - # we can blow this away when we're done importing. - shutil.rmtree(course_dir) - - logging.debug('new course at {0}'.format(course_items[0].location)) - - create_all_course_groups(request.user, course_items[0].location) - - logging.debug('created all course groups at {0}'.format(course_items[0].location)) - - return HttpResponse(json.dumps({'Status': 'OK'})) - else: - course_module = modulestore().get_item(location) - - return render_to_response('import.html', { - 'context_course': course_module, - 'successful_import_redirect_url': reverse('course_index', kwargs={ - 'org': location.org, - 'course': location.course, - 'name': location.name, - }) - }) - - -@ensure_csrf_cookie -@login_required -def generate_export_course(request, org, course, name): - """ - This method will serialize out a course to a .tar.gz file which contains a XML-based representation of - the course - """ - location = get_location_and_verify_access(request, org, course, name) - course_module = modulestore().get_instance(location.course_id, location) - loc = Location(location) - export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") - - root_dir = path(mkdtemp()) - - try: - export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name, modulestore()) - except SerializationError, e: - logging.exception('There was an error exporting course {0}. {1}'.format(course_module.location, unicode(e))) - - unit = None - failed_item = None - parent = None - try: - failed_item = modulestore().get_instance(course_module.location.course_id, e.location) - parent_locs = modulestore().get_parent_locations(failed_item.location, course_module.location.course_id) - - if len(parent_locs) > 0: - parent = modulestore().get_item(parent_locs[0]) - if parent.location.category == 'vertical': - unit = parent - except: - # if we have a nested exception, then we'll show the more generic error message - pass - - return render_to_response('export.html', { - 'context_course': course_module, - 'successful_import_redirect_url': '', - 'in_err': True, - 'raw_err_msg': str(e), - 'failed_module': failed_item, - 'unit': unit, - 'edit_unit_url': reverse('edit_unit', kwargs={ - 'location': parent.location - }) if parent else '', - 'course_home_url': reverse('course_index', kwargs={ - 'org': org, - 'course': course, - 'name': name - }) - }) - except Exception, e: - logging.exception('There was an error exporting course {0}. {1}'.format(course_module.location, unicode(e))) - return render_to_response('export.html', { - 'context_course': course_module, - 'successful_import_redirect_url': '', - 'in_err': True, - 'unit': None, - 'raw_err_msg': str(e), - 'course_home_url': reverse('course_index', kwargs={ - 'org': org, - 'course': course, - 'name': name - }) - }) - - logging.debug('tar file being generated at {0}'.format(export_file.name)) - tar_file = tarfile.open(name=export_file.name, mode='w:gz') - tar_file.add(root_dir / name, arcname=name) - tar_file.close() - - # remove temp dir - shutil.rmtree(root_dir / name) - - wrapper = FileWrapper(export_file) - response = HttpResponse(wrapper, content_type='application/x-tgz') - response['Content-Disposition'] = 'attachment; filename=%s' % os.path.basename(export_file.name) - response['Content-Length'] = os.path.getsize(export_file.name) - return response - - -@ensure_csrf_cookie -@login_required -def export_course(request, org, course, name): - """ - This method serves up the 'Export Course' page - """ - location = get_location_and_verify_access(request, org, course, name) - - course_module = modulestore().get_item(location) - - return render_to_response('export.html', { - 'context_course': course_module, - 'successful_import_redirect_url': '' - }) diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py index aad56e4a2e..939286a765 100644 --- a/cms/djangoapps/contentstore/views/course.py +++ b/cms/djangoapps/contentstore/views/course.py @@ -18,6 +18,7 @@ from mitxmako.shortcuts import render_to_response from xmodule.modulestore.django import modulestore from xmodule.modulestore.inheritance import own_metadata +from xmodule.contentstore.content import StaticContent from xmodule.modulestore.exceptions import ( ItemNotFoundError, InvalidLocationError) @@ -206,7 +207,8 @@ def course_info(request, org, course, name, provided_id=None): 'context_course': course_module, 'url_base': "/" + org + "/" + course + "/", 'course_updates': json.dumps(get_course_updates(location)), - 'handouts_location': Location(['i4x', org, course, 'course_info', 'handouts']).url() }) + 'handouts_location': Location(['i4x', org, course, 'course_info', 'handouts']).url(), + 'base_asset_url': StaticContent.get_base_url_path_for_course_assets(location) + '/'}) @expect_json diff --git a/cms/djangoapps/contentstore/views/import_export.py b/cms/djangoapps/contentstore/views/import_export.py new file mode 100644 index 0000000000..5830e07a52 --- /dev/null +++ b/cms/djangoapps/contentstore/views/import_export.py @@ -0,0 +1,325 @@ +""" +These views handle all actions in Studio related to import and exporting of +courses +""" +import logging +import os +import tarfile +import shutil +import re +from tempfile import mkdtemp +from path import path +from contextlib import contextmanager + +from django.conf import settings +from django.http import HttpResponse +from django.contrib.auth.decorators import login_required +from django_future.csrf import ensure_csrf_cookie +from django.core.urlresolvers import reverse +from django.core.servers.basehttp import FileWrapper +from django.core.files.temp import NamedTemporaryFile +from django.views.decorators.http import require_http_methods + +from mitxmako.shortcuts import render_to_response +from auth.authz import create_all_course_groups + +from xmodule.modulestore.xml_importer import import_from_xml +from xmodule.contentstore.django import contentstore +from xmodule.modulestore.xml_exporter import export_to_xml +from xmodule.modulestore.django import modulestore +from xmodule.modulestore import Location +from xmodule.exceptions import SerializationError + +from .access import get_location_and_verify_access +from util.json_request import JsonResponse + + +__all__ = ['import_course', 'generate_export_course', 'export_course'] + +log = logging.getLogger(__name__) + + +# Regex to capture Content-Range header ranges. +CONTENT_RE = re.compile(r"(?P\d{1,11})-(?P\d{1,11})/(?P\d{1,11})") + + +@ensure_csrf_cookie +@require_http_methods(("GET", "POST", "PUT")) +@login_required +def import_course(request, org, course, name): + """ + This method will handle a POST request to upload and import a .tar.gz file + into a specified course + """ + location = get_location_and_verify_access(request, org, course, name) + + @contextmanager + def wfile(filename, dirname): + """ + A with-context that creates `filename` on entry and removes it on exit. + `filename` is truncted on creation. Additionally removes dirname on + exit. + """ + open("file", "w").close() + try: + yield filename + finally: + os.remove(filename) + shutil.rmtree(dirname) + + if request.method == 'POST': + + data_root = path(settings.GITHUB_REPO_ROOT) + course_subdir = "{0}-{1}-{2}".format(org, course, name) + course_dir = data_root / course_subdir + + filename = request.FILES['course-data'].name + if not filename.endswith('.tar.gz'): + return JsonResponse( + {'ErrMsg': 'We only support uploading a .tar.gz file.'}, + status=415 + ) + temp_filepath = course_dir / filename + + if not course_dir.isdir(): + os.mkdir(course_dir) + + logging.debug('importing course to {0}'.format(temp_filepath)) + + # Get upload chunks byte ranges + try: + matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"]) + content_range = matches.groupdict() + except KeyError: # Single chunk + # no Content-Range header, so make one that will work + content_range = {'start': 0, 'stop': 1, 'end': 2} + + # stream out the uploaded files in chunks to disk + if int(content_range['start']) == 0: + mode = "wb+" + else: + mode = "ab+" + size = os.path.getsize(temp_filepath) + # Check to make sure we haven't missed a chunk + # This shouldn't happen, even if different instances are handling + # the same session, but it's always better to catch errors earlier. + if size < int(content_range['start']): + log.warning( + "Reported range %s does not match size downloaded so far %s", + content_range['start'], + size + ) + return JsonResponse( + {'ErrMsg': 'File upload corrupted. Please try again'}, + status=409 + ) + # The last request sometimes comes twice. This happens because + # nginx sends a 499 error code when the response takes too long. + elif size > int(content_range['stop']) and size == int(content_range['end']): + return JsonResponse({'ImportStatus': 1}) + + with open(temp_filepath, mode) as temp_file: + for chunk in request.FILES['course-data'].chunks(): + temp_file.write(chunk) + + size = os.path.getsize(temp_filepath) + + if int(content_range['stop']) != int(content_range['end']) - 1: + # More chunks coming + return JsonResponse({ + "files": [{ + "name": filename, + "size": size, + "deleteUrl": "", + "deleteType": "", + "url": reverse('import_course', kwargs={ + 'org': location.org, + 'course': location.course, + 'name': location.name + }), + "thumbnailUrl": "" + }] + }) + + else: # This was the last chunk. + + # 'Lock' with status info. + status_file = data_root / (course + filename + ".lock") + + # Do everything from now on in a with-context, to be sure we've + # properly cleaned up. + with wfile(status_file, course_dir): + + with open(status_file, 'w+') as sf: + sf.write("Extracting") + + tar_file = tarfile.open(temp_filepath) + tar_file.extractall(course_dir + '/') + + with open(status_file, 'w+') as sf: + sf.write("Verifying") + + # find the 'course.xml' file + dirpath = None + + def get_all_files(directory): + """ + For each file in the directory, yield a 2-tuple of (file-name, + directory-path) + """ + for dirpath, _dirnames, filenames in os.walk(directory): + for filename in filenames: + yield (filename, dirpath) + + def get_dir_for_fname(directory, filename): + """ + Returns the dirpath for the first file found in the directory + with the given name. If there is no file in the directory with + the specified name, return None. + """ + for fname, dirpath in get_all_files(directory): + if fname == filename: + return dirpath + return None + + fname = "course.xml" + + dirpath = get_dir_for_fname(course_dir, fname) + + if not dirpath: + return JsonResponse( + {'ErrMsg': 'Could not find the course.xml file in the package.'}, + status=415 + ) + + logging.debug('found course.xml at {0}'.format(dirpath)) + + if dirpath != course_dir: + for fname in os.listdir(dirpath): + shutil.move(dirpath / fname, course_dir) + + _module_store, course_items = import_from_xml( + modulestore('direct'), + settings.GITHUB_REPO_ROOT, + [course_subdir], + load_error_modules=False, + static_content_store=contentstore(), + target_location_namespace=location, + draft_store=modulestore() + ) + + logging.debug('new course at {0}'.format(course_items[0].location)) + + with open(status_file, 'w') as sf: + sf.write("Updating course") + + create_all_course_groups(request.user, course_items[0].location) + logging.debug('created all course groups at {0}'.format(course_items[0].location)) + + return JsonResponse({'Status': 'OK'}) + else: + course_module = modulestore().get_item(location) + + return render_to_response('import.html', { + 'context_course': course_module, + 'successful_import_redirect_url': reverse('course_index', kwargs={ + 'org': location.org, + 'course': location.course, + 'name': location.name, + }) + }) + + +@ensure_csrf_cookie +@login_required +def generate_export_course(request, org, course, name): + """ + This method will serialize out a course to a .tar.gz file which contains a + XML-based representation of the course + """ + location = get_location_and_verify_access(request, org, course, name) + course_module = modulestore().get_instance(location.course_id, location) + loc = Location(location) + export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") + + root_dir = path(mkdtemp()) + + try: + export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name, modulestore()) + except SerializationError, e: + logging.exception('There was an error exporting course {0}. {1}'.format(course_module.location, unicode(e))) + unit = None + failed_item = None + parent = None + try: + failed_item = modulestore().get_instance(course_module.location.course_id, e.location) + parent_locs = modulestore().get_parent_locations(failed_item.location, course_module.location.course_id) + + if len(parent_locs) > 0: + parent = modulestore().get_item(parent_locs[0]) + if parent.location.category == 'vertical': + unit = parent + except: + # if we have a nested exception, then we'll show the more generic error message + pass + + return render_to_response('export.html', { + 'context_course': course_module, + 'successful_import_redirect_url': '', + 'in_err': True, + 'raw_err_msg': str(e), + 'failed_module': failed_item, + 'unit': unit, + 'edit_unit_url': reverse('edit_unit', kwargs={ + 'location': parent.location + }) if parent else '', + 'course_home_url': reverse('course_index', kwargs={ + 'org': org, + 'course': course, + 'name': name + }) + }) + except Exception, e: + logging.exception('There was an error exporting course {0}. {1}'.format(course_module.location, unicode(e))) + return render_to_response('export.html', { + 'context_course': course_module, + 'successful_import_redirect_url': '', + 'in_err': True, + 'unit': None, + 'raw_err_msg': str(e), + 'course_home_url': reverse('course_index', kwargs={ + 'org': org, + 'course': course, + 'name': name + }) + }) + + logging.debug('tar file being generated at {0}'.format(export_file.name)) + tar_file = tarfile.open(name=export_file.name, mode='w:gz') + tar_file.add(root_dir / name, arcname=name) + tar_file.close() + + # remove temp dir + shutil.rmtree(root_dir / name) + + wrapper = FileWrapper(export_file) + response = HttpResponse(wrapper, content_type='application/x-tgz') + response['Content-Disposition'] = 'attachment; filename=%s' % os.path.basename(export_file.name) + response['Content-Length'] = os.path.getsize(export_file.name) + return response + + +@ensure_csrf_cookie +@login_required +def export_course(request, org, course, name): + """ + This method serves up the 'Export Course' page + """ + location = get_location_and_verify_access(request, org, course, name) + + course_module = modulestore().get_item(location) + + return render_to_response('export.html', { + 'context_course': course_module, + 'successful_import_redirect_url': '' + }) diff --git a/cms/envs/common.py b/cms/envs/common.py index a06d5a36e1..4421474287 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -246,7 +246,7 @@ PIPELINE_JS = { 'js/models/metadata_model.js', 'js/views/metadata_editor_view.js', 'js/models/uploads.js', 'js/views/uploads.js', 'js/models/textbook.js', 'js/views/textbook.js', - 'js/views/assets.js', 'js/utility.js', + 'js/views/assets.js', 'js/src/utility.js', 'js/models/settings/course_grading_policy.js'], 'output_filename': 'js/cms-application.js', 'test_order': 0 diff --git a/cms/envs/jasmine.py b/cms/envs/jasmine.py deleted file mode 100644 index a4b8292d71..0000000000 --- a/cms/envs/jasmine.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -This configuration is used for running jasmine tests -""" - -# We intentionally define lots of variables that aren't used, and -# want to import all variables from base settings files -# pylint: disable=W0401, W0614 - -from .test import * -from logsettings import get_logger_config - -ENABLE_JASMINE = True -DEBUG = True - -LOGGING = get_logger_config(TEST_ROOT / "log", - logging_env="dev", - tracking_filename="tracking.log", - dev_env=True, - debug=True, - local_loglevel='ERROR', - console_loglevel='ERROR') - -PIPELINE_JS['js-test-source'] = { - 'source_filenames': sum([ - pipeline_group['source_filenames'] - for group_name, pipeline_group - in sorted(PIPELINE_JS.items(), key=lambda item: item[1].get('test_order', 1e100)) - if group_name != 'spec' - ], []), - 'output_filename': 'js/cms-test-source.js' -} - -PIPELINE_JS['spec'] = { - 'source_filenames': sorted(rooted_glob(PROJECT_ROOT / 'static/', 'coffee/spec/**/*.js')), - 'output_filename': 'js/cms-spec.js' -} - -JASMINE_TEST_DIRECTORY = PROJECT_ROOT + '/static/coffee' -JASMINE_REPORT_DIR = os.environ.get('JASMINE_REPORT_DIR', 'reports/cms/jasmine') - -TEMPLATE_CONTEXT_PROCESSORS += ('settings_context_processor.context_processors.settings',) -TEMPLATE_VISIBLE_SETTINGS = ('JASMINE_REPORT_DIR', ) - -STATICFILES_DIRS.append(REPO_ROOT/'node_modules/phantom-jasmine/lib') -STATICFILES_DIRS.append(REPO_ROOT/'node_modules/jasmine-reporters/src') - -# Remove the localization middleware class because it requires the test database -# to be sync'd and migrated in order to run the jasmine tests interactively -# with a browser -MIDDLEWARE_CLASSES = tuple(e for e in MIDDLEWARE_CLASSES \ - if e != 'django.middleware.locale.LocaleMiddleware') - -INSTALLED_APPS += ('django_jasmine', 'settings_context_processor') diff --git a/cms/one_time_startup.py b/cms/one_time_startup.py deleted file mode 100644 index 4198cf2637..0000000000 --- a/cms/one_time_startup.py +++ /dev/null @@ -1,22 +0,0 @@ -from dogapi import dog_http_api, dog_stats_api -from django.conf import settings -from xmodule.modulestore.django import modulestore -from django.dispatch import Signal -from request_cache.middleware import RequestCache - -from django.core.cache import get_cache - -CACHE = get_cache('mongo_metadata_inheritance') -for store_name in settings.MODULESTORE: - store = modulestore(store_name) - - store.set_modulestore_configuration({ - 'metadata_inheritance_cache_subsystem': CACHE, - 'request_cache': RequestCache.get_request_cache() - }) - - modulestore_update_signal = Signal(providing_args=['modulestore', 'course_id', 'location']) - store.modulestore_update_signal = modulestore_update_signal -if hasattr(settings, 'DATADOG_API'): - dog_http_api.api_key = settings.DATADOG_API - dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True) diff --git a/cms/startup.py b/cms/startup.py new file mode 100644 index 0000000000..eb1098a707 --- /dev/null +++ b/cms/startup.py @@ -0,0 +1,25 @@ +""" +Module with code executed during Studio startup +""" +from django.conf import settings + +# Force settings to run so that the python path is modified +settings.INSTALLED_APPS # pylint: disable=W0104 + +from django_startup import autostartup + +# TODO: Remove this code once Studio/CMS runs via wsgi in all environments +INITIALIZED = False + + +def run(): + """ + Executed during django startup + """ + global INITIALIZED + if INITIALIZED: + return + + INITIALIZED = True + autostartup() + diff --git a/cms/static/coffee/files.json b/cms/static/coffee/files.json deleted file mode 100644 index 3964bee455..0000000000 --- a/cms/static/coffee/files.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "static_files": [ - "../jsi18n/", - "js/vendor/RequireJS.js", - "js/vendor/jquery.min.js", - "js/vendor/jquery-ui.min.js", - "js/vendor/jquery.ui.draggable.js", - "js/vendor/jquery.cookie.js", - "js/vendor/json2.js", - "js/vendor/underscore-min.js", - "js/vendor/underscore.string.min.js", - "js/vendor/backbone-min.js", - "js/vendor/backbone-associations-min.js", - "js/vendor/jquery.leanModal.min.js", - "js/vendor/jquery.form.js", - "js/vendor/sinon-1.7.1.js", - "js/vendor/jasmine-stealth.js", - "js/test/i18n.js" - ] -} diff --git a/cms/static/coffee/fixtures b/cms/static/coffee/fixtures new file mode 120000 index 0000000000..800ce1d60d --- /dev/null +++ b/cms/static/coffee/fixtures @@ -0,0 +1 @@ +../../templates/js/ \ No newline at end of file diff --git a/cms/static/coffee/fixtures/edit-chapter.underscore b/cms/static/coffee/fixtures/edit-chapter.underscore deleted file mode 120000 index 9e057ab233..0000000000 --- a/cms/static/coffee/fixtures/edit-chapter.underscore +++ /dev/null @@ -1 +0,0 @@ -../../../templates/js/edit-chapter.underscore \ No newline at end of file diff --git a/cms/static/coffee/fixtures/edit-textbook.underscore b/cms/static/coffee/fixtures/edit-textbook.underscore deleted file mode 120000 index 5bb17a2d43..0000000000 --- a/cms/static/coffee/fixtures/edit-textbook.underscore +++ /dev/null @@ -1 +0,0 @@ -../../../templates/js/edit-textbook.underscore \ No newline at end of file diff --git a/cms/static/coffee/fixtures/metadata-editor.underscore b/cms/static/coffee/fixtures/metadata-editor.underscore deleted file mode 120000 index 9696774d0a..0000000000 --- a/cms/static/coffee/fixtures/metadata-editor.underscore +++ /dev/null @@ -1 +0,0 @@ -../../../templates/js/metadata-editor.underscore \ No newline at end of file diff --git a/cms/static/coffee/fixtures/metadata-list-entry.underscore b/cms/static/coffee/fixtures/metadata-list-entry.underscore deleted file mode 120000 index 78fa4e2000..0000000000 --- a/cms/static/coffee/fixtures/metadata-list-entry.underscore +++ /dev/null @@ -1 +0,0 @@ -../../../templates/js/metadata-list-entry.underscore \ No newline at end of file diff --git a/cms/static/coffee/fixtures/metadata-number-entry.underscore b/cms/static/coffee/fixtures/metadata-number-entry.underscore deleted file mode 120000 index 99138aa9c1..0000000000 --- a/cms/static/coffee/fixtures/metadata-number-entry.underscore +++ /dev/null @@ -1 +0,0 @@ -../../../templates/js/metadata-number-entry.underscore \ No newline at end of file diff --git a/cms/static/coffee/fixtures/metadata-option-entry.underscore b/cms/static/coffee/fixtures/metadata-option-entry.underscore deleted file mode 120000 index c6cd499801..0000000000 --- a/cms/static/coffee/fixtures/metadata-option-entry.underscore +++ /dev/null @@ -1 +0,0 @@ -../../../templates/js/metadata-option-entry.underscore \ No newline at end of file diff --git a/cms/static/coffee/fixtures/metadata-string-entry.underscore b/cms/static/coffee/fixtures/metadata-string-entry.underscore deleted file mode 120000 index f713ab5387..0000000000 --- a/cms/static/coffee/fixtures/metadata-string-entry.underscore +++ /dev/null @@ -1 +0,0 @@ -../../../templates/js/metadata-string-entry.underscore \ No newline at end of file diff --git a/cms/static/coffee/fixtures/no-textbooks.underscore b/cms/static/coffee/fixtures/no-textbooks.underscore deleted file mode 120000 index d2e1c9a71a..0000000000 --- a/cms/static/coffee/fixtures/no-textbooks.underscore +++ /dev/null @@ -1 +0,0 @@ -../../../templates/js/no-textbooks.underscore \ No newline at end of file diff --git a/cms/static/coffee/fixtures/section-name-edit.underscore b/cms/static/coffee/fixtures/section-name-edit.underscore deleted file mode 120000 index 89284ccf90..0000000000 --- a/cms/static/coffee/fixtures/section-name-edit.underscore +++ /dev/null @@ -1 +0,0 @@ -../../../templates/js/section-name-edit.underscore \ No newline at end of file diff --git a/cms/static/coffee/fixtures/show-textbook.underscore b/cms/static/coffee/fixtures/show-textbook.underscore deleted file mode 120000 index d2ba37d689..0000000000 --- a/cms/static/coffee/fixtures/show-textbook.underscore +++ /dev/null @@ -1 +0,0 @@ -../../../templates/js/show-textbook.underscore \ No newline at end of file diff --git a/cms/static/coffee/fixtures/system-feedback.underscore b/cms/static/coffee/fixtures/system-feedback.underscore deleted file mode 120000 index 10893f87c4..0000000000 --- a/cms/static/coffee/fixtures/system-feedback.underscore +++ /dev/null @@ -1 +0,0 @@ -../../../templates/js/system-feedback.underscore \ No newline at end of file diff --git a/cms/static/coffee/fixtures/upload-dialog.underscore b/cms/static/coffee/fixtures/upload-dialog.underscore deleted file mode 120000 index e5637e9a53..0000000000 --- a/cms/static/coffee/fixtures/upload-dialog.underscore +++ /dev/null @@ -1 +0,0 @@ -../../../templates/js/upload-dialog.underscore \ No newline at end of file diff --git a/cms/static/coffee/spec/helpers.coffee b/cms/static/coffee/spec/helpers.coffee index 116983edf5..a03e2a0e56 100644 --- a/cms/static/coffee/spec/helpers.coffee +++ b/cms/static/coffee/spec/helpers.coffee @@ -1,4 +1,4 @@ -jasmine.getFixtures().fixturesPath = 'fixtures' +jasmine.getFixtures().fixturesPath += 'coffee/fixtures' # Stub jQuery.cookie @stubCookies = diff --git a/cms/static/coffee/spec/views/course_info_spec.coffee b/cms/static/coffee/spec/views/course_info_spec.coffee new file mode 100644 index 0000000000..297e78f34a --- /dev/null +++ b/cms/static/coffee/spec/views/course_info_spec.coffee @@ -0,0 +1,144 @@ +courseInfoPage = """ +
+
+
+
    +
    +
    + +
    + """ + +commonSetup = () -> + window.analytics = jasmine.createSpyObj('analytics', ['track']) + window.course_location_analytics = jasmine.createSpy() + window.courseUpdatesXhr = sinon.useFakeXMLHttpRequest() + requests = [] + window.courseUpdatesXhr.onCreate = (xhr) -> requests.push(xhr) + return requests + +commonCleanup = () -> + window.courseUpdatesXhr.restore() + delete window.analytics + delete window.course_location_analytics + +describe "Course Updates", -> + courseInfoTemplate = readFixtures('course_info_update.underscore') + + beforeEach -> + setFixtures($(" + + diff --git a/cms/urls.py b/cms/urls.py index 8f396d3742..0f5209173b 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -1,9 +1,9 @@ from django.conf import settings from django.conf.urls import patterns, include, url -# Import this file so it can do its work, even though we don't use the name. -# pylint: disable=W0611 -from . import one_time_startup +# TODO: This should be removed once the CMS is running via wsgi on all production servers +import cms.startup as startup +startup.run() # There is a course creators admin table. from ratelimitbackend import admin @@ -135,10 +135,6 @@ urlpatterns += ( url(r'^jsi18n/$', 'django.views.i18n.javascript_catalog', js_info_dict), ) - -if settings.ENABLE_JASMINE: - urlpatterns += (url(r'^_jasmine/', include('django_jasmine.urls')),) - if settings.MITX_FEATURES.get('ENABLE_SERVICE_STATUS'): urlpatterns += ( url(r'^status/', include('service_status.urls')), diff --git a/cms/wsgi.py b/cms/wsgi.py new file mode 100644 index 0000000000..607d7ee709 --- /dev/null +++ b/cms/wsgi.py @@ -0,0 +1,12 @@ +import os + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cms.envs.aws") + +import cms.startup as startup +startup.run() + +# This application object is used by the development server +# as well as any WSGI server configured to use this file. +from django.core.wsgi import get_wsgi_application +application = get_wsgi_application() + diff --git a/common/djangoapps/course_groups/tests/tests.py b/common/djangoapps/course_groups/tests/tests.py index 2e519edb30..a17df56a71 100644 --- a/common/djangoapps/course_groups/tests/tests.py +++ b/common/djangoapps/course_groups/tests/tests.py @@ -8,19 +8,20 @@ from course_groups.models import CourseUserGroup from course_groups.cohorts import (get_cohort, get_course_cohorts, is_commentable_cohorted, get_cohort_by_name) -from xmodule.modulestore.django import modulestore, _MODULESTORES +from xmodule.modulestore.django import modulestore, clear_existing_modulestores -from xmodule.modulestore.tests.django_utils import xml_store_config +from xmodule.modulestore.tests.django_utils import mixed_store_config # NOTE: running this with the lms.envs.test config works without # manually overriding the modulestore. However, running with # cms.envs.test doesn't. TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT -TEST_DATA_XML_MODULESTORE = xml_store_config(TEST_DATA_DIR) +TEST_MAPPING = {'edX/toy/2012_Fall': 'xml'} +TEST_DATA_MIXED_MODULESTORE = mixed_store_config(TEST_DATA_DIR, TEST_MAPPING) -@override_settings(MODULESTORE=TEST_DATA_XML_MODULESTORE) +@override_settings(MODULESTORE=TEST_DATA_MIXED_MODULESTORE) class TestCohorts(django.test.TestCase): @staticmethod @@ -82,9 +83,7 @@ class TestCohorts(django.test.TestCase): """ Make sure that course is reloaded every time--clear out the modulestore. """ - # don't like this, but don't know a better way to undo all changes made - # to course. We don't have a course.clone() method. - _MODULESTORES.clear() + clear_existing_modulestores() def test_get_cohort(self): """ diff --git a/common/djangoapps/course_modes/models.py b/common/djangoapps/course_modes/models.py index eff944d450..d5f0ba7503 100644 --- a/common/djangoapps/course_modes/models.py +++ b/common/djangoapps/course_modes/models.py @@ -57,11 +57,6 @@ class CourseMode(models.Model): def modes_for_course_dict(cls, course_id): return { mode.slug : mode for mode in cls.modes_for_course(course_id) } - def __unicode__(self): - return u"{} : {}, min={}, prices={}".format( - self.course_id, self.mode_slug, self.min_price, self.suggested_prices - ) - @classmethod def mode_for_course(cls, course_id, mode_slug): """ @@ -76,3 +71,8 @@ class CourseMode(models.Model): return matched[0] else: return None + + def __unicode__(self): + return u"{} : {}, min={}, prices={}".format( + self.course_id, self.mode_slug, self.min_price, self.suggested_prices + ) diff --git a/common/djangoapps/datadog/startup.py b/common/djangoapps/datadog/startup.py new file mode 100644 index 0000000000..41949c3a94 --- /dev/null +++ b/common/djangoapps/datadog/startup.py @@ -0,0 +1,12 @@ +from django.conf import settings +from dogapi import dog_http_api, dog_stats_api + +def run(): + """ + Initialize connection to datadog during django startup. + + Expects the datadog api key in the DATADOG_API settings key + """ + if hasattr(settings, 'DATADOG_API'): + dog_http_api.api_key = settings.DATADOG_API + dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True) diff --git a/common/djangoapps/external_auth/tests/test_shib.py b/common/djangoapps/external_auth/tests/test_shib.py index 6bb9c38e6f..0355730256 100644 --- a/common/djangoapps/external_auth/tests/test_shib.py +++ b/common/djangoapps/external_auth/tests/test_shib.py @@ -14,11 +14,9 @@ from django.contrib.auth.models import AnonymousUser, User from django.utils.importlib import import_module from xmodule.modulestore.tests.factories import CourseFactory -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, mixed_store_config from xmodule.modulestore.inheritance import own_metadata -from xmodule.modulestore.django import modulestore - -from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE +from xmodule.modulestore.django import editable_modulestore from external_auth.models import ExternalAuthMap from external_auth.views import shib_login, course_specific_login, course_specific_register @@ -27,6 +25,8 @@ from student.views import create_account, change_enrollment from student.models import UserProfile, Registration, CourseEnrollment from student.tests.factories import UserFactory +TEST_DATA_MIXED_MODULESTORE = mixed_store_config(settings.COMMON_TEST_DATA_ROOT, {}) + # Shib is supposed to provide 'REMOTE_USER', 'givenName', 'sn', 'mail', 'Shib-Identity-Provider' # attributes via request.META. We can count on 'Shib-Identity-Provider', and 'REMOTE_USER' being present # b/c of how mod_shib works but should test the behavior with the rest of the attributes present/missing @@ -64,7 +64,7 @@ def gen_all_identities(): yield _build_identity_dict(mail, given_name, surname) -@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE, SESSION_ENGINE='django.contrib.sessions.backends.cache') +@override_settings(MODULESTORE=TEST_DATA_MIXED_MODULESTORE, SESSION_ENGINE='django.contrib.sessions.backends.cache') class ShibSPTest(ModuleStoreTestCase): """ Tests for the Shibboleth SP, which communicates via request.META @@ -73,7 +73,7 @@ class ShibSPTest(ModuleStoreTestCase): request_factory = RequestFactory() def setUp(self): - self.store = modulestore() + self.store = editable_modulestore() @unittest.skipUnless(settings.MITX_FEATURES.get('AUTH_USE_SHIB'), True) def test_exception_shib_login(self): diff --git a/common/djangoapps/mitxmako/middleware.py b/common/djangoapps/mitxmako/middleware.py index 5646d2f4b5..daaddf6b87 100644 --- a/common/djangoapps/mitxmako/middleware.py +++ b/common/djangoapps/mitxmako/middleware.py @@ -12,36 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from mako.lookup import TemplateLookup -import tempdir from django.template import RequestContext -from django.conf import settings requestcontext = None -lookup = {} - class MakoMiddleware(object): - def __init__(self): - """Setup mako variables and lookup object""" - # Set all mako variables based on django settings - template_locations = settings.MAKO_TEMPLATES - module_directory = getattr(settings, 'MAKO_MODULE_DIR', None) - - if module_directory is None: - module_directory = tempdir.mkdtemp_clean() - - for location in template_locations: - lookup[location] = TemplateLookup(directories=template_locations[location], - module_directory=module_directory, - output_encoding='utf-8', - input_encoding='utf-8', - default_filters=['decode.utf8'], - encoding_errors='replace', - ) - - import mitxmako - mitxmako.lookup = lookup def process_request(self, request): global requestcontext diff --git a/common/djangoapps/mitxmako/shortcuts.py b/common/djangoapps/mitxmako/shortcuts.py index 3c68fa85be..974eaefdd3 100644 --- a/common/djangoapps/mitxmako/shortcuts.py +++ b/common/djangoapps/mitxmako/shortcuts.py @@ -16,7 +16,7 @@ from django.template import Context from django.http import HttpResponse import logging -from . import middleware +import mitxmako from django.conf import settings from django.core.urlresolvers import reverse log = logging.getLogger(__name__) @@ -80,15 +80,15 @@ def render_to_string(template_name, dictionary, context=None, namespace='main'): context_instance['marketing_link'] = marketing_link # In various testing contexts, there might not be a current request context. - if middleware.requestcontext is not None: - for d in middleware.requestcontext: + if mitxmako.middleware.requestcontext is not None: + for d in mitxmako.middleware.requestcontext: context_dictionary.update(d) for d in context_instance: context_dictionary.update(d) if context: context_dictionary.update(context) # fetch and render template - template = middleware.lookup[namespace].get_template(template_name) + template = mitxmako.lookup[namespace].get_template(template_name) return template.render_unicode(**context_dictionary) diff --git a/common/djangoapps/mitxmako/startup.py b/common/djangoapps/mitxmako/startup.py new file mode 100644 index 0000000000..db9483b366 --- /dev/null +++ b/common/djangoapps/mitxmako/startup.py @@ -0,0 +1,33 @@ +""" +Initialize the mako template lookup +""" + +import tempdir +from django.conf import settings +from mako.lookup import TemplateLookup + +import mitxmako + + +def run(): + """Setup mako variables and lookup object""" + # Set all mako variables based on django settings + template_locations = settings.MAKO_TEMPLATES + module_directory = getattr(settings, 'MAKO_MODULE_DIR', None) + + if module_directory is None: + module_directory = tempdir.mkdtemp_clean() + + lookup = {} + + for location in template_locations: + lookup[location] = TemplateLookup( + directories=template_locations[location], + module_directory=module_directory, + output_encoding='utf-8', + input_encoding='utf-8', + default_filters=['decode.utf8'], + encoding_errors='replace', + ) + + mitxmako.lookup = lookup diff --git a/common/djangoapps/mitxmako/template.py b/common/djangoapps/mitxmako/template.py index 5becfbf1df..7dfc6de026 100644 --- a/common/djangoapps/mitxmako/template.py +++ b/common/djangoapps/mitxmako/template.py @@ -16,7 +16,8 @@ from django.conf import settings from mako.template import Template as MakoTemplate from mitxmako.shortcuts import marketing_link -from mitxmako import middleware +import mitxmako +import mitxmako.middleware django_variables = ['lookup', 'output_encoding', 'encoding_errors'] @@ -33,7 +34,7 @@ class Template(MakoTemplate): def __init__(self, *args, **kwargs): """Overrides base __init__ to provide django variable overrides""" if not kwargs.get('no_django', False): - overrides = dict([(k, getattr(middleware, k, None),) for k in django_variables]) + overrides = dict([(k, getattr(mitxmako, k, None),) for k in django_variables]) overrides['lookup'] = overrides['lookup']['main'] kwargs.update(overrides) super(Template, self).__init__(*args, **kwargs) @@ -47,8 +48,8 @@ class Template(MakoTemplate): context_dictionary = {} # In various testing contexts, there might not be a current request context. - if middleware.requestcontext is not None: - for d in middleware.requestcontext: + if mitxmako.middleware.requestcontext is not None: + for d in mitxmako.middleware.requestcontext: context_dictionary.update(d) for d in context_instance: context_dictionary.update(d) diff --git a/common/djangoapps/static_replace/test/test_static_replace.py b/common/djangoapps/static_replace/test/test_static_replace.py index b1bc05b895..43a199c22c 100644 --- a/common/djangoapps/static_replace/test/test_static_replace.py +++ b/common/djangoapps/static_replace/test/test_static_replace.py @@ -1,6 +1,6 @@ import re -from nose.tools import assert_equals, assert_true, assert_false +from nose.tools import assert_equals, assert_true, assert_false # pylint: disable=E0611 from static_replace import (replace_static_urls, replace_course_urls, _url_replace_regex) from mock import patch, Mock diff --git a/common/djangoapps/student/management/commands/6002exportusers.py b/common/djangoapps/student/management/commands/6002exportusers.py index a92bb0a60c..a36d6e84c6 100644 --- a/common/djangoapps/student/management/commands/6002exportusers.py +++ b/common/djangoapps/student/management/commands/6002exportusers.py @@ -16,10 +16,6 @@ from django.contrib.auth.models import User from student.models import UserProfile -import mitxmako.middleware as middleware - -middleware.MakoMiddleware() - class Command(BaseCommand): help = \ diff --git a/common/djangoapps/student/management/commands/6002importusers.py b/common/djangoapps/student/management/commands/6002importusers.py index 1f98bd7522..93d86dfc05 100644 --- a/common/djangoapps/student/management/commands/6002importusers.py +++ b/common/djangoapps/student/management/commands/6002importusers.py @@ -12,10 +12,6 @@ from django.contrib.auth.models import User from student.models import UserProfile -import mitxmako.middleware as middleware - -middleware.MakoMiddleware() - def import_user(u): user_info = u['u'] diff --git a/common/djangoapps/student/management/commands/assigngroups.py b/common/djangoapps/student/management/commands/assigngroups.py index 3e36bf3129..cbd5cfad22 100644 --- a/common/djangoapps/student/management/commands/assigngroups.py +++ b/common/djangoapps/student/management/commands/assigngroups.py @@ -1,7 +1,6 @@ from django.core.management.base import BaseCommand from django.contrib.auth.models import User -import mitxmako.middleware as middleware from student.models import UserTestGroup import random @@ -11,8 +10,6 @@ import datetime import json from pytz import UTC -middleware.MakoMiddleware() - def group_from_value(groups, v): ''' Given group: (('a',0.3),('b',0.4),('c',0.3)) And random value diff --git a/common/djangoapps/student/management/commands/emaillist.py b/common/djangoapps/student/management/commands/emaillist.py index d3911927ac..e69b072db2 100644 --- a/common/djangoapps/student/management/commands/emaillist.py +++ b/common/djangoapps/student/management/commands/emaillist.py @@ -1,10 +1,6 @@ from django.core.management.base import BaseCommand from django.contrib.auth.models import User -import mitxmako.middleware as middleware - -middleware.MakoMiddleware() - class Command(BaseCommand): help = \ diff --git a/common/djangoapps/student/management/commands/massemail.py b/common/djangoapps/student/management/commands/massemail.py index 1bb65fd169..a1864f048e 100644 --- a/common/djangoapps/student/management/commands/massemail.py +++ b/common/djangoapps/student/management/commands/massemail.py @@ -1,9 +1,7 @@ from django.core.management.base import BaseCommand from django.contrib.auth.models import User -import mitxmako.middleware as middleware - -middleware.MakoMiddleware() +import mitxmako class Command(BaseCommand): @@ -17,8 +15,8 @@ body, and an _subject.txt for the subject. ''' #text = open(args[0]).read() #subject = open(args[1]).read() users = User.objects.all() - text = middleware.lookup['main'].get_template('email/' + args[0] + ".txt").render() - subject = middleware.lookup['main'].get_template('email/' + args[0] + "_subject.txt").render().strip() + text = mitxmako.lookup['main'].get_template('email/' + args[0] + ".txt").render() + subject = mitxmako.lookup['main'].get_template('email/' + args[0] + "_subject.txt").render().strip() for user in users: if user.is_active: user.email_user(subject, text) diff --git a/common/djangoapps/student/management/commands/massemailtxt.py b/common/djangoapps/student/management/commands/massemailtxt.py index ae25430a85..0228acf923 100644 --- a/common/djangoapps/student/management/commands/massemailtxt.py +++ b/common/djangoapps/student/management/commands/massemailtxt.py @@ -4,15 +4,13 @@ import time from django.core.management.base import BaseCommand from django.conf import settings -import mitxmako.middleware as middleware +import mitxmako from django.core.mail import send_mass_mail import sys import datetime -middleware.MakoMiddleware() - def chunks(l, n): """ Yield successive n-sized chunks from l. @@ -41,8 +39,8 @@ rate -- messages per second users = [u.strip() for u in open(user_file).readlines()] - message = middleware.lookup['main'].get_template('emails/' + message_base + "_body.txt").render() - subject = middleware.lookup['main'].get_template('emails/' + message_base + "_subject.txt").render().strip() + message = mitxmako.lookup['main'].get_template('emails/' + message_base + "_body.txt").render() + subject = mitxmako.lookup['main'].get_template('emails/' + message_base + "_subject.txt").render().strip() rate = int(ratestr) self.log_file = open(logfilename, "a+", buffering=0) diff --git a/common/djangoapps/student/management/commands/userinfo.py b/common/djangoapps/student/management/commands/userinfo.py index 5467db1733..8656fb9183 100644 --- a/common/djangoapps/student/management/commands/userinfo.py +++ b/common/djangoapps/student/management/commands/userinfo.py @@ -1,13 +1,10 @@ from django.core.management.base import BaseCommand from django.contrib.auth.models import User -import mitxmako.middleware as middleware import json from student.models import UserProfile -middleware.MakoMiddleware() - class Command(BaseCommand): help = \ diff --git a/common/djangoapps/student/models.py b/common/djangoapps/student/models.py index 3d977b28c9..5f29ffa6aa 100644 --- a/common/djangoapps/student/models.py +++ b/common/djangoapps/student/models.py @@ -805,7 +805,8 @@ class CourseEnrollment(models.Model): record.is_active = False record.save() except cls.DoesNotExist: - log.error("Tried to unenroll student {} from {} but they were not enrolled") + err_msg = u"Tried to unenroll student {} from {} but they were not enrolled" + log.error(err_msg.format(user, course_id)) @classmethod def unenroll_by_email(cls, email, course_id): diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py index 9173eb8224..7b02cc37d5 100644 --- a/common/djangoapps/student/views.py +++ b/common/djangoapps/student/views.py @@ -1,3 +1,6 @@ +""" +Student Views +""" import datetime import json import logging @@ -52,6 +55,10 @@ from courseware.access import has_access from external_auth.models import ExternalAuthMap +from bulk_email.models import Optout + +import track.views + from statsd import statsd from pytz import UTC @@ -62,8 +69,7 @@ Article = namedtuple('Article', 'title url author image deck publication publish def csrf_token(context): - ''' A csrf token that can be included in a form. - ''' + """A csrf token that can be included in a form.""" csrf_token = context.get('csrf_token', '') if csrf_token == 'NOTPROVIDED': return '' @@ -76,12 +82,12 @@ def csrf_token(context): # This means that it should always return the same thing for anon # users. (in particular, no switching based on query params allowed) def index(request, extra_context={}, user=None): - ''' + """ Render the edX main page. extra_context is used to allow immediate display of certain modal windows, eg signup, as used by external_auth. - ''' + """ # The course selection work is done in courseware.courses. domain = settings.MITX_FEATURES.get('FORCE_UNIVERSITY_DOMAIN') # normally False @@ -265,6 +271,8 @@ def dashboard(request): log.error("User {0} enrolled in non-existent course {1}" .format(user.username, enrollment.course_id)) + course_optouts = Optout.objects.filter(user=user).values_list('course_id', flat=True) + message = "" if not user.is_active: message = render_to_string('registration/activate_account_notice.html', {'email': user.email}) @@ -292,6 +300,7 @@ def dashboard(request): pass context = {'courses': courses, + 'course_optouts': course_optouts, 'message': message, 'external_auth_map': external_auth_map, 'staff_access': staff_access, @@ -411,7 +420,7 @@ def accounts_login(request, error=""): # Need different levels of logging @ensure_csrf_cookie def login_user(request, error=""): - ''' AJAX request to log in the user. ''' + """AJAX request to log in the user.""" if 'email' not in request.POST or 'password' not in request.POST: return HttpResponse(json.dumps({'success': False, 'value': _('There was an error receiving your login information. Please email us.')})) # TODO: User error message @@ -494,11 +503,11 @@ def login_user(request, error=""): @ensure_csrf_cookie def logout_user(request): - ''' + """ HTTP request to log out the user. Redirects to marketing page. Deletes both the CSRF and sessionid cookies so the marketing site can determine the logged in state of the user - ''' + """ # We do not log here, because we have a handler registered # to perform logging on successful logouts. logout(request) @@ -512,8 +521,7 @@ def logout_user(request): @login_required @ensure_csrf_cookie def change_setting(request): - ''' JSON call to change a profile setting: Right now, location - ''' + """JSON call to change a profile setting: Right now, location""" # TODO (vshnayder): location is no longer used up = UserProfile.objects.get(user=request.user) # request.user.profile_cache if 'location' in request.POST: @@ -581,10 +589,10 @@ def _do_create_account(post_vars): @ensure_csrf_cookie def create_account(request, post_override=None): - ''' + """ JSON call to create new edX account. Used by form in signup_modal.html, which is included into navigation.html - ''' + """ js = {'success': False} post_vars = post_override if post_override else request.POST @@ -818,10 +826,10 @@ def begin_exam_registration(request, course_id): @ensure_csrf_cookie def create_exam_registration(request, post_override=None): - ''' + """ JSON call to create a test center exam registration. Called by form in test_center_register.html - ''' + """ post_vars = post_override if post_override else request.POST # first determine if we need to create a new TestCenterUser, or if we are making any update @@ -974,8 +982,7 @@ def auto_auth(request): @ensure_csrf_cookie def activate_account(request, key): - ''' When link in activation e-mail is clicked - ''' + """When link in activation e-mail is clicked""" r = Registration.objects.filter(activation_key=key) if len(r) == 1: user_logged_in = request.user.is_authenticated() @@ -1010,7 +1017,7 @@ def activate_account(request, key): @ensure_csrf_cookie def password_reset(request): - ''' Attempts to send a password reset e-mail. ''' + """ Attempts to send a password reset e-mail. """ if request.method != "POST": raise Http404 @@ -1032,9 +1039,9 @@ def password_reset_confirm_wrapper( uidb36=None, token=None, ): - ''' A wrapper around django.contrib.auth.views.password_reset_confirm. + """ A wrapper around django.contrib.auth.views.password_reset_confirm. Needed because we want to set the user as active at this step. - ''' + """ # cribbed from django.contrib.auth.views.password_reset_confirm try: uid_int = base36_to_int(uidb36) @@ -1076,8 +1083,8 @@ def reactivation_email_for_user(user): @ensure_csrf_cookie def change_email_request(request): - ''' AJAX call from the profile page. User wants a new e-mail. - ''' + """ AJAX call from the profile page. User wants a new e-mail. + """ ## Make sure it checks for existing e-mail conflicts if not request.user.is_authenticated: raise Http404 @@ -1132,9 +1139,9 @@ def change_email_request(request): @ensure_csrf_cookie @transaction.commit_manually def confirm_email_change(request, key): - ''' User requested a new e-mail. This is called when the activation + """ User requested a new e-mail. This is called when the activation link is clicked. We confirm with the old e-mail, and update - ''' + """ try: try: pec = PendingEmailChange.objects.get(activation_key=key) @@ -1191,7 +1198,7 @@ def confirm_email_change(request, key): @ensure_csrf_cookie def change_name_request(request): - ''' Log a request for a new name. ''' + """ Log a request for a new name. """ if not request.user.is_authenticated: raise Http404 @@ -1215,7 +1222,7 @@ def change_name_request(request): @ensure_csrf_cookie def pending_name_changes(request): - ''' Web page which allows staff to approve or reject name changes. ''' + """ Web page which allows staff to approve or reject name changes. """ if not request.user.is_staff: raise Http404 @@ -1231,7 +1238,7 @@ def pending_name_changes(request): @ensure_csrf_cookie def reject_name_change(request): - ''' JSON: Name change process. Course staff clicks 'reject' on a given name change ''' + """ JSON: Name change process. Course staff clicks 'reject' on a given name change """ if not request.user.is_staff: raise Http404 @@ -1269,13 +1276,36 @@ def accept_name_change_by_id(id): @ensure_csrf_cookie def accept_name_change(request): - ''' JSON: Name change process. Course staff clicks 'accept' on a given name change + """ JSON: Name change process. Course staff clicks 'accept' on a given name change We used this during the prototype but now we simply record name changes instead of manually approving them. Still keeping this around in case we want to go back to this approval method. - ''' + """ if not request.user.is_staff: raise Http404 return accept_name_change_by_id(int(request.POST['id'])) + + +@require_POST +@login_required +@ensure_csrf_cookie +def change_email_settings(request): + """Modify logged-in user's setting for receiving emails from a course.""" + user = request.user + + course_id = request.POST.get("course_id") + receive_emails = request.POST.get("receive_emails") + if receive_emails: + optout_object = Optout.objects.filter(user=user, course_id=course_id) + if optout_object: + optout_object.delete() + log.info(u"User {0} ({1}) opted in to receive emails from course {2}".format(user.username, user.email, course_id)) + track.views.server_track(request, "change-email-settings", {"receive_emails": "yes", "course": course_id}, page='dashboard') + else: + Optout.objects.get_or_create(user=user, course_id=course_id) + log.info(u"User {0} ({1}) opted out of receiving emails from course {2}".format(user.username, user.email, course_id)) + track.views.server_track(request, "change-email-settings", {"receive_emails": "no", "course": course_id}, page='dashboard') + + return HttpResponse(json.dumps({'success': True})) diff --git a/common/djangoapps/terrain/browser.py b/common/djangoapps/terrain/browser.py index cf53aa4f69..75c0764b1b 100644 --- a/common/djangoapps/terrain/browser.py +++ b/common/djangoapps/terrain/browser.py @@ -16,11 +16,6 @@ from requests import put from base64 import encodestring from json import dumps -# Let the LMS and CMS do their one-time setup -# For example, setting up mongo caches -# These names aren't used, but do important work on import. -from lms import one_time_startup # pylint: disable=W0611 -from cms import one_time_startup # pylint: disable=W0611 from pymongo import MongoClient import xmodule.modulestore.django from xmodule.contentstore.django import _CONTENTSTORE @@ -161,9 +156,10 @@ def reset_databases(scenario): mongo = MongoClient() mongo.drop_database(settings.CONTENTSTORE['OPTIONS']['db']) _CONTENTSTORE.clear() - modulestore = xmodule.modulestore.django.modulestore() + + modulestore = xmodule.modulestore.django.editable_modulestore() modulestore.collection.drop() - xmodule.modulestore.django._MODULESTORES.clear() + xmodule.modulestore.django.clear_existing_modulestores() # Uncomment below to trigger a screenshot on error diff --git a/common/djangoapps/terrain/course_helpers.py b/common/djangoapps/terrain/course_helpers.py index eca3290080..fc01d25d66 100644 --- a/common/djangoapps/terrain/course_helpers.py +++ b/common/djangoapps/terrain/course_helpers.py @@ -10,7 +10,7 @@ from django.contrib.auth import authenticate, login from django.contrib.auth.middleware import AuthenticationMiddleware from django.contrib.sessions.middleware import SessionMiddleware from student.models import CourseEnrollment -from xmodule.modulestore.django import modulestore +from xmodule.modulestore.django import editable_modulestore from xmodule.contentstore.django import contentstore from urllib import quote_plus @@ -60,11 +60,9 @@ def register_by_course_id(course_id, is_staff=False): @world.absorb def clear_courses(): # Flush and initialize the module store - # It needs the templates because it creates new records - # by cloning from the template. # Note that if your test module gets in some weird state # (though it shouldn't), do this manually # from the bash shell to drop it: # $ mongo test_xmodule --eval "db.dropDatabase()" - modulestore().collection.drop() + editable_modulestore().collection.drop() contentstore().fs_files.drop() diff --git a/common/djangoapps/terrain/steps.py b/common/djangoapps/terrain/steps.py index f13b3ff932..c4783d4aca 100644 --- a/common/djangoapps/terrain/steps.py +++ b/common/djangoapps/terrain/steps.py @@ -15,7 +15,7 @@ from lettuce import world, step from .course_helpers import * from .ui_helpers import * from lettuce.django import django_url -from nose.tools import assert_equals +from nose.tools import assert_equals # pylint: disable=E0611 from logging import getLogger logger = getLogger(__name__) diff --git a/common/djangoapps/terrain/ui_helpers.py b/common/djangoapps/terrain/ui_helpers.py index 3ab7e11b47..7d308931b2 100644 --- a/common/djangoapps/terrain/ui_helpers.py +++ b/common/djangoapps/terrain/ui_helpers.py @@ -10,7 +10,7 @@ from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from lettuce.django import django_url -from nose.tools import assert_true +from nose.tools import assert_true # pylint: disable=E0611 @world.absorb diff --git a/common/lib/calc/calc/__init__.py b/common/lib/calc/calc/__init__.py new file mode 100644 index 0000000000..e0d80d7b89 --- /dev/null +++ b/common/lib/calc/calc/__init__.py @@ -0,0 +1,6 @@ +""" +Ideally, we wouldn't need to pull in all the calc symbols here, +but courses were using 'import calc', so we need this for +backwards compatibility +""" +from calc import * diff --git a/common/lib/calc/calc.py b/common/lib/calc/calc/calc.py similarity index 96% rename from common/lib/calc/calc.py rename to common/lib/calc/calc/calc.py index ab300f121b..efb30f93d7 100644 --- a/common/lib/calc/calc.py +++ b/common/lib/calc/calc/calc.py @@ -9,7 +9,7 @@ import operator import numbers import numpy import scipy.constants -import calcfunctions +import functions from pyparsing import ( Word, Literal, CaselessLiteral, ZeroOrMore, MatchFirst, Optional, Forward, @@ -20,9 +20,9 @@ DEFAULT_FUNCTIONS = { 'sin': numpy.sin, 'cos': numpy.cos, 'tan': numpy.tan, - 'sec': calcfunctions.sec, - 'csc': calcfunctions.csc, - 'cot': calcfunctions.cot, + 'sec': functions.sec, + 'csc': functions.csc, + 'cot': functions.cot, 'sqrt': numpy.sqrt, 'log10': numpy.log10, 'log2': numpy.log2, @@ -31,24 +31,24 @@ DEFAULT_FUNCTIONS = { 'arccos': numpy.arccos, 'arcsin': numpy.arcsin, 'arctan': numpy.arctan, - 'arcsec': calcfunctions.arcsec, - 'arccsc': calcfunctions.arccsc, - 'arccot': calcfunctions.arccot, + 'arcsec': functions.arcsec, + 'arccsc': functions.arccsc, + 'arccot': functions.arccot, 'abs': numpy.abs, 'fact': math.factorial, 'factorial': math.factorial, 'sinh': numpy.sinh, 'cosh': numpy.cosh, 'tanh': numpy.tanh, - 'sech': calcfunctions.sech, - 'csch': calcfunctions.csch, - 'coth': calcfunctions.coth, + 'sech': functions.sech, + 'csch': functions.csch, + 'coth': functions.coth, 'arcsinh': numpy.arcsinh, 'arccosh': numpy.arccosh, 'arctanh': numpy.arctanh, - 'arcsech': calcfunctions.arcsech, - 'arccsch': calcfunctions.arccsch, - 'arccoth': calcfunctions.arccoth + 'arcsech': functions.arcsech, + 'arccsch': functions.arccsch, + 'arccoth': functions.arccoth } DEFAULT_VARIABLES = { 'i': numpy.complex(0, 1), diff --git a/common/lib/calc/calcfunctions.py b/common/lib/calc/calc/functions.py similarity index 100% rename from common/lib/calc/calcfunctions.py rename to common/lib/calc/calc/functions.py diff --git a/common/lib/calc/preview.py b/common/lib/calc/calc/preview.py similarity index 100% rename from common/lib/calc/preview.py rename to common/lib/calc/calc/preview.py diff --git a/common/lib/calc/tests/test_calc.py b/common/lib/calc/calc/tests/test_calc.py similarity index 100% rename from common/lib/calc/tests/test_calc.py rename to common/lib/calc/calc/tests/test_calc.py diff --git a/common/lib/calc/tests/test_preview.py b/common/lib/calc/calc/tests/test_preview.py similarity index 99% rename from common/lib/calc/tests/test_preview.py rename to common/lib/calc/calc/tests/test_preview.py index 0008cdda47..7db307a742 100644 --- a/common/lib/calc/tests/test_preview.py +++ b/common/lib/calc/calc/tests/test_preview.py @@ -4,7 +4,7 @@ Unit tests for preview.py """ import unittest -import preview +from calc import preview import pyparsing diff --git a/common/lib/calc/setup.py b/common/lib/calc/setup.py index cb638914f9..361884babf 100644 --- a/common/lib/calc/setup.py +++ b/common/lib/calc/setup.py @@ -2,8 +2,8 @@ from setuptools import setup setup( name="calc", - version="0.1.1", - py_modules=["calc"], + version="0.2", + packages=["calc"], install_requires=[ "pyparsing==1.5.6", "numpy", diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index c2bdeadc21..08a223f609 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -555,6 +555,13 @@ class LoncapaProblem(object): Used by get_html. ''' + if not isinstance(problemtree.tag, basestring): + # Comment and ProcessingInstruction nodes are not Elements, + # and we're ok leaving those behind. + # BTW: etree gives us no good way to distinguish these things + # other than to examine .tag to see if it's a string. :( + return + if (problemtree.tag == 'script' and problemtree.get('type') and 'javascript' in problemtree.get('type')): # leave javascript intact. diff --git a/common/lib/capa/capa/inputtypes.py b/common/lib/capa/capa/inputtypes.py index 9defd2c5e6..d27893d44d 100644 --- a/common/lib/capa/capa/inputtypes.py +++ b/common/lib/capa/capa/inputtypes.py @@ -49,7 +49,7 @@ import pyparsing from .registry import TagRegistry from chem import chemcalc -from preview import latex_preview +from calc.preview import latex_preview import xqueue_interface from datetime import datetime diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 731230ecc1..b53f38fd90 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -915,7 +915,26 @@ class NumericalResponse(LoncapaResponse): else: return CorrectMap(self.answer_id, 'incorrect') - # TODO: add check_hint_condition(self, hxml_set, student_answers) + def compare_answer(self, ans1, ans2): + """ + Outside-facing function that lets us compare two numerical answers, + with this problem's tolerance. + """ + return compare_with_tolerance( + evaluator({}, {}, ans1), + evaluator({}, {}, ans2), + self.tolerance + ) + + def validate_answer(self, answer): + """ + Returns whether this answer is in a valid form. + """ + try: + evaluator(dict(), dict(), answer) + return True + except (StudentInputError, UndefinedVariable): + return False def get_answers(self): return {self.answer_id: self.correct_answer} @@ -1778,46 +1797,24 @@ class FormulaResponse(LoncapaResponse): self.correct_answer, given, self.samples) return CorrectMap(self.answer_id, correctness) - def check_formula(self, expected, given, samples): - variables = samples.split('@')[0].split(',') - numsamples = int(samples.split('@')[1].split('#')[1]) - sranges = zip(*map(lambda x: map(float, x.split(",")), - samples.split('@')[1].split('#')[0].split(':'))) - - ranges = dict(zip(variables, sranges)) - for _ in range(numsamples): - instructor_variables = self.strip_dict(dict(self.context)) - student_variables = {} - # ranges give numerical ranges for testing - for var in ranges: - # TODO: allow specified ranges (i.e. integers and complex numbers) for random variables - value = random.uniform(*ranges[var]) - instructor_variables[str(var)] = value - student_variables[str(var)] = value - # log.debug('formula: instructor_vars=%s, expected=%s' % - # (instructor_variables,expected)) - - # Call `evaluator` on the instructor's answer and get a number - instructor_result = evaluator( - instructor_variables, {}, - expected, case_sensitive=self.case_sensitive - ) + def tupleize_answers(self, answer, var_dict_list): + """ + Takes in an answer and a list of dictionaries mapping variables to values. + Each dictionary represents a test case for the answer. + Returns a tuple of formula evaluation results. + """ + out = [] + for var_dict in var_dict_list: try: - # log.debug('formula: student_vars=%s, given=%s' % - # (student_variables,given)) - - # Call `evaluator` on the student's answer; look for exceptions - student_result = evaluator( - student_variables, - {}, - given, - case_sensitive=self.case_sensitive - ) + out.append(evaluator( + var_dict, + dict(), + answer, + case_sensitive=self.case_sensitive, + )) except UndefinedVariable as uv: log.debug( - 'formularesponse: undefined variable in given=%s', - given - ) + 'formularesponse: undefined variable in formula=%s' % answer) raise StudentInputError( "Invalid input: " + uv.message + " not permitted in answer" ) @@ -1840,17 +1837,70 @@ class FormulaResponse(LoncapaResponse): # If non-factorial related ValueError thrown, handle it the same as any other Exception log.debug('formularesponse: error {0} in formula'.format(ve)) raise StudentInputError("Invalid input: Could not parse '%s' as a formula" % - cgi.escape(given)) + cgi.escape(answer)) except Exception as err: # traceback.print_exc() log.debug('formularesponse: error %s in formula', err) raise StudentInputError("Invalid input: Could not parse '%s' as a formula" % - cgi.escape(given)) + cgi.escape(answer)) + return out - # No errors in student's response--actually test for correctness - if not compare_with_tolerance(student_result, instructor_result, self.tolerance): - return "incorrect" - return "correct" + def randomize_variables(self, samples): + """ + Returns a list of dictionaries mapping variables to random values in range, + as expected by tupleize_answers. + """ + variables = samples.split('@')[0].split(',') + numsamples = int(samples.split('@')[1].split('#')[1]) + sranges = zip(*map(lambda x: map(float, x.split(",")), + samples.split('@')[1].split('#')[0].split(':'))) + ranges = dict(zip(variables, sranges)) + + out = [] + for i in range(numsamples): + var_dict = {} + # ranges give numerical ranges for testing + for var in ranges: + # TODO: allow specified ranges (i.e. integers and complex numbers) for random variables + value = random.uniform(*ranges[var]) + var_dict[str(var)] = value + out.append(var_dict) + return out + + def check_formula(self, expected, given, samples): + """ + Given an expected answer string, a given (student-produced) answer + string, and a samples string, return whether the given answer is + "correct" or "incorrect". + """ + var_dict_list = self.randomize_variables(samples) + student_result = self.tupleize_answers(given, var_dict_list) + instructor_result = self.tupleize_answers(expected, var_dict_list) + + correct = all(compare_with_tolerance(student, instructor, self.tolerance) + for student, instructor in zip(student_result, instructor_result)) + if correct: + return "correct" + else: + return "incorrect" + + def compare_answer(self, ans1, ans2): + """ + An external interface for comparing whether a and b are equal. + """ + internal_result = self.check_formula(ans1, ans2, self.samples) + return internal_result == "correct" + + def validate_answer(self, answer): + """ + Returns whether this answer is in a valid form. + """ + var_dict_list = self.randomize_variables(self.samples) + try: + self.tupleize_answers(answer, var_dict_list) + return True + except StudentInputError: + return False def strip_dict(self, d): ''' Takes a dict. Returns an identical dict, with all non-word diff --git a/common/lib/capa/capa/safe_exec/README.rst b/common/lib/capa/capa/safe_exec/README.rst index c61100f709..00b81ca15f 100644 --- a/common/lib/capa/capa/safe_exec/README.rst +++ b/common/lib/capa/capa/safe_exec/README.rst @@ -16,11 +16,11 @@ __ https://github.com/edx/codejail/blob/master/README.rst 1. At the instruction to install packages into the sandboxed code, you'll - need to install both `pre-sandbox-requirements.txt` and - `sandbox-requirements.txt`:: + need to install the requirements from requirements/edx-sandbox:: - $ sudo pip install -r pre-sandbox-requirements.txt - $ sudo pip install -r sandbox-requirements.txt + $ pip install -r requirements/edx-sandbox/base.txt + $ pip install -r requirements/edx-sandbox/local.txt + $ pip install -r requirements/edx-sandbox/post.txt 2. At the instruction to create the AppArmor profile, you'll need a line in the profile for the sandbox packages. is the full path to diff --git a/common/lib/capa/capa/templates/choicegroup.html b/common/lib/capa/capa/templates/choicegroup.html index 17f7efcec4..cc153efe3c 100644 --- a/common/lib/capa/capa/templates/choicegroup.html +++ b/common/lib/capa/capa/templates/choicegroup.html @@ -17,6 +17,7 @@ % for choice_id, choice_description in choices: