diff --git a/.gitignore b/.gitignore index d01baf055a..76cc1efa95 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ *.swp *.orig *.DS_Store +*.mo :2e_* :2e# .AppleDouble @@ -22,6 +23,8 @@ reports/ *.egg-info Gemfile.lock .env/ +conf/locale/en/LC_MESSAGES/*.po +!messages.po lms/static/sass/*.css cms/static/sass/*.css lms/lib/comment_client/python @@ -33,3 +36,7 @@ chromedriver.log /nbproject ghostdriver.log node_modules +.pip_download_cache/ +.prereqs_cache +autodeploy.properties +.ws_migrations_complete diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 253bae3686..0000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "common/test/phantom-jasmine"] - path = common/test/phantom-jasmine - url = https://github.com/jcarver989/phantom-jasmine.git \ No newline at end of file diff --git a/.tx/config b/.tx/config new file mode 100644 index 0000000000..540c4732af --- /dev/null +++ b/.tx/config @@ -0,0 +1,26 @@ +[main] +host = https://www.transifex.com + +[edx-studio.django-partial] +file_filter = conf/locale//LC_MESSAGES/django-partial.po +source_file = conf/locale/en/LC_MESSAGES/django-partial.po +source_lang = en +type = PO + +[edx-studio.djangojs] +file_filter = conf/locale//LC_MESSAGES/djangojs.po +source_file = conf/locale/en/LC_MESSAGES/djangojs.po +source_lang = en +type = PO + +[edx-studio.mako] +file_filter = conf/locale//LC_MESSAGES/mako.po +source_file = conf/locale/en/LC_MESSAGES/mako.po +source_lang = en +type = PO + +[edx-studio.messages] +file_filter = conf/locale//LC_MESSAGES/messages.po +source_file = conf/locale/en/LC_MESSAGES/messages.po +source_lang = en +type = PO diff --git a/LICENSE.TXT b/LICENSE similarity index 100% rename from LICENSE.TXT rename to LICENSE diff --git a/README b/README deleted file mode 100644 index 2ed50ba063..0000000000 --- a/README +++ /dev/null @@ -1 +0,0 @@ -See doc/ for documentation. diff --git a/README.md b/README.md new file mode 100644 index 0000000000..90b82ff07a --- /dev/null +++ b/README.md @@ -0,0 +1,153 @@ +This is edX, a platform for online course delivery. The project is primarily +written in [Python](http://python.org/), using the +[Django](https://www.djangoproject.com/) framework. We also use some +[Ruby](http://www.ruby-lang.org/) and some [NodeJS](http://nodejs.org/). + +Installation +============ +The installation process is a bit messy at the moment. Here's a high-level +overview of what you should do to get started. + +**TLDR:** There is a `scripts/create-dev-env.sh` script that will attempt to set all +of this up for you. If you're in a hurry, run that script. Otherwise, I suggest +that you understand what the script is doing, and why, by reading this document. + +Directory Hierarchy +------------------- +This code assumes that it is checked out in a directory that has three sibling +directories: `data` (used for XML course data), `db` (used to hold a +[sqlite](https://sqlite.org/) database), and `log` (used to hold logs). If you +clone the repository into a directory called `edx` inside of a directory +called `dev`, here's an example of how the directory hierarchy should look: + + * dev + \ + * data + * db + * log + * edx + \ + README.md + +Language Runtimes +----------------- +You'll need to be sure that you have Python 2.7, Ruby 1.9.3, and NodeJS +(latest stable) installed on your system. Some of these you can install +using your system's package manager: [homebrew](http://mxcl.github.io/homebrew/) +for Mac, [apt](http://wiki.debian.org/Apt) for Debian-based systems +(including Ubuntu), [rpm](http://www.rpm.org/) or [yum](http://yum.baseurl.org/) +for Red Hat based systems (including CentOS). + +If your system's package manager gives you the wrong version of a language +runtime, then you'll need to use a versioning tool to install the correct version. +Usually, you'll need to do this for Ruby: you can use +[`rbenv`](https://github.com/sstephenson/rbenv) or [`rvm`](https://rvm.io/), but +typically `rbenv` is simpler. For Python, you can use +[`pythonz`](http://saghul.github.io/pythonz/), +and for Node, you can use [`nvm`](https://github.com/creationix/nvm). + +Virtual Environments +-------------------- +Often, different projects will have conflicting dependencies: for example, two +projects depending on two different, incompatible versions of a library. Clearly, +you can't have both versions installed and used on your machine simultaneously. +Virtual environments were created to solve this problem: by installing libraries +into an isolated environment, only projects that live inside the environment +will be able to see and use those libraries. Got incompatible dependencies? Use +different virtual environments, and your problem is solved. + +Remember, each language has a different implementation. Python has +[`virtualenv`](http://www.virtualenv.org/), Ruby has +[`bundler`](http://gembundler.com/), and Node's virtual environment support +is built into [`npm`](https://npmjs.org/), its library management tool. +For each language, decide if you want to use a virtual environment, or if you +want to install all the language dependencies globally (and risk conflicts). +I suggest you start with installing things globally until and unless things +break; you can always switch over to a virtual environment later on. + +Language Packages +----------------- +The Python libraries we use are listed in `requirements.txt`. The Ruby libraries +we use are listed in `Gemfile`. The Node libraries we use are listed in +`packages.json`. Python has a library installer called +[`pip`](http://www.pip-installer.org/), Ruby has a library installer called +[`gem`](https://rubygems.org/) (or `bundle` if you're using a virtual +environment), and Node has a library installer called +[`npm`](https://npmjs.org/). +Once you've got your languages and virtual environments set up, install +the libraries like so: + + $ pip install -r requirements/base.txt + $ pip install -r requirements/post.txt + $ bundle install + $ npm install + +You can also use [`rake`](http://rake.rubyforge.org/) to get all of the prerequisites (or to update) +them if they've changed + + $ rake install_prereqs + +Other Dependencies +------------------ +You'll also need to install [MongoDB](http://www.mongodb.org/), since our +application uses it in addition to sqlite. You can install it through your +system package manager, and I suggest that you configure it to start +automatically when you boot up your system, so that you never have to worry +about it again. For Mac, use +[`launchd`](https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man8/launchd.8.html) +(running `brew info mongodb` will give you some commands you can copy-paste.) +For Linux, you can use [`upstart`](http://upstart.ubuntu.com/), `chkconfig`, +or any other process management tool. + +Configuring Your Project +------------------------ +We use [`rake`](http://rake.rubyforge.org/) to execute common tasks in our +project. The `rake` tasks are defined in the `rakefile`, or you can run `rake -T` +to view a summary. + +Before you run your project, you need to create a sqlite database, create +tables in that database, run database migrations, and populate templates for +CMS templates. Fortunately, `rake` will do all of this for you! Just run: + + $ rake django-admin[syncdb] + $ rake django-admin[migrate] + $ rake django-admin[update_templates] + +If you are running these commands using the [`zsh`](http://www.zsh.org/) shell, +zsh will assume that you are doing +[shell globbing](https://en.wikipedia.org/wiki/Glob_(programming)), search for +a file in your directory named `django-adminsyncdb` or `django-adminmigrate`, +and fail. To fix this, just surround the argument with quotation marks, so that +you're running `rake "django-admin[syncdb]"`. + +Run Your Project +---------------- +edX has two components: Studio, the course authoring system; and the LMS +(learning management system) used by students. These two systems communicate +through the MongoDB database, which stores course information. + +To run Studio, run: + + $ rake cms + +To run the LMS, run: + + $ rake lms[cms.dev] + +Studio runs on port 8001, while LMS runs on port 8000, so you can run both of +these commands simultaneously, using two different terminal windows. To view +Studio, visit `127.0.0.1:8001` in your web browser; to view the LMS, visit +`127.0.0.1:8000`. + +There's also an older version of the LMS that saves its information in XML files +in the `data` directory, instead of in Mongo. To run this older version, run: + + $ rake lms + +Further Documentation +===================== +Once you've got your project up and running, you can check out the `docs` +directory to see more documentation about how edX is structured. + + + diff --git a/cms/djangoapps/contentstore/features/advanced-settings.feature b/cms/djangoapps/contentstore/features/advanced-settings.feature index ca5b62e596..6f6cc50702 100644 --- a/cms/djangoapps/contentstore/features/advanced-settings.feature +++ b/cms/djangoapps/contentstore/features/advanced-settings.feature @@ -11,7 +11,8 @@ Feature: Advanced (manual) course policy Given I am on the Advanced Course Settings page in Studio Then the settings are alphabetized - @skip-phantom + # Skipped because Ubuntu ChromeDriver cannot click notification "Cancel" + @skip Scenario: Test cancel editing key value Given I am on the Advanced Course Settings page in Studio When I edit the value of a policy key @@ -20,7 +21,8 @@ Feature: Advanced (manual) course policy And I reload the page Then the policy key value is unchanged - @skip-phantom + # Skipped because Ubuntu ChromeDriver cannot click notification "Save" + @skip Scenario: Test editing key value Given I am on the Advanced Course Settings page in Studio When I edit the value of a policy key and save @@ -28,7 +30,8 @@ Feature: Advanced (manual) course policy And I reload the page Then the policy key value is changed - @skip-phantom + # Skipped because Ubuntu ChromeDriver cannot edit CodeMirror input + @skip Scenario: Test how multi-line input appears Given I am on the Advanced Course Settings page in Studio When I create a JSON object as a value @@ -36,7 +39,8 @@ Feature: Advanced (manual) course policy And I reload the page Then it is displayed as formatted - @skip-phantom + # Skipped because Ubuntu ChromeDriver cannot edit CodeMirror input + @skip Scenario: Test automatic quoting of non-JSON values Given I am on the Advanced Course Settings page in Studio When I create a non-JSON value not in quotes diff --git a/cms/djangoapps/contentstore/features/checklists.feature b/cms/djangoapps/contentstore/features/checklists.feature index ddf1adf263..3767144c99 100644 --- a/cms/djangoapps/contentstore/features/checklists.feature +++ b/cms/djangoapps/contentstore/features/checklists.feature @@ -10,8 +10,6 @@ Feature: Course checklists Then I can check and uncheck tasks in a checklist And They are correctly selected after I reload the page - @skip-phantom - @skip-firefox Scenario: A task can link to a location within Studio Given I have opened Checklists When I select a link to the course outline @@ -19,8 +17,6 @@ Feature: Course checklists And I press the browser back button Then I am brought back to the course outline in the correct state - @skip-phantom - @skip-firefox Scenario: A task can link to a location outside Studio Given I have opened Checklists When I select a link to help page diff --git a/cms/djangoapps/contentstore/features/checklists.py b/cms/djangoapps/contentstore/features/checklists.py index d433dbbf0d..1c9fbf0994 100644 --- a/cms/djangoapps/contentstore/features/checklists.py +++ b/cms/djangoapps/contentstore/features/checklists.py @@ -2,7 +2,7 @@ #pylint: disable=W0621 from lettuce import world, step -from nose.tools import assert_true, assert_equal +from nose.tools import assert_true, assert_equal, assert_in from terrain.steps import reload_the_page from selenium.common.exceptions import StaleElementReferenceException @@ -63,7 +63,7 @@ def i_select_a_link_to_the_course_outline(step): @step('I am brought to the course outline page$') def i_am_brought_to_course_outline(step): - assert_equal('Course Outline', world.css_find('.outline .title-1')[0].text) + assert_in('Course Outline', world.css_find('.outline .page-header')[0].text) assert_equal(1, len(world.browser.windows)) diff --git a/cms/djangoapps/contentstore/features/course-settings.feature b/cms/djangoapps/contentstore/features/course-settings.feature index fc9641cb46..e869bfe47a 100644 --- a/cms/djangoapps/contentstore/features/course-settings.feature +++ b/cms/djangoapps/contentstore/features/course-settings.feature @@ -1,20 +1,17 @@ Feature: Course Settings As a course author, I want to be able to configure my course settings. - @skip-phantom Scenario: User can set course dates Given I have opened a new course in Studio When I select Schedule and Details And I set course dates Then I see the set dates on refresh - @skip-phantom Scenario: User can clear previously set course dates (except start date) Given I have set course dates And I clear all the dates except start Then I see cleared dates on refresh - @skip-phantom Scenario: User cannot clear the course start date Given I have set course dates And I clear the course start date diff --git a/cms/djangoapps/contentstore/features/section.feature b/cms/djangoapps/contentstore/features/section.feature index 24cbeb3db9..236cf501fc 100644 --- a/cms/djangoapps/contentstore/features/section.feature +++ b/cms/djangoapps/contentstore/features/section.feature @@ -3,7 +3,6 @@ Feature: Create Section As a course author I want to create and edit sections - @skip-phantom Scenario: Add a new section to a course Given I have opened a new course in Studio When I click the New Section link @@ -27,7 +26,8 @@ Feature: Create Section And I save a new section release date Then the section release date is updated - @skip-phantom + # Skipped because Ubuntu ChromeDriver hangs on alert + @skip Scenario: Delete section Given I have opened a new course in Studio And I have added a new section diff --git a/cms/djangoapps/contentstore/features/signup.py b/cms/djangoapps/contentstore/features/signup.py index 6ca358183b..398f8d074d 100644 --- a/cms/djangoapps/contentstore/features/signup.py +++ b/cms/djangoapps/contentstore/features/signup.py @@ -18,10 +18,7 @@ def i_fill_in_the_registration_form(step): @step('I press the Create My Account button on the registration form$') def i_press_the_button_on_the_registration_form(step): submit_css = 'form#register_form button#submit' - # Workaround for click not working on ubuntu - # for some unknown reason. - e = world.css_find(submit_css) - e.type(' ') + world.css_click(submit_css) @step('I should see be on the studio home page$') diff --git a/cms/djangoapps/contentstore/features/studio-overview-togglesection.feature b/cms/djangoapps/contentstore/features/studio-overview-togglesection.feature index a0e0a48f9e..c9f5b43dfb 100644 --- a/cms/djangoapps/contentstore/features/studio-overview-togglesection.feature +++ b/cms/djangoapps/contentstore/features/studio-overview-togglesection.feature @@ -14,7 +14,6 @@ Feature: Overview Toggle Section When I navigate to the course overview page Then I do not see the "Collapse All Sections" link - @skip-phantom Scenario: Collapse link appears after creating first section of a course Given I have a course with no sections When I navigate to the course overview page @@ -22,7 +21,8 @@ Feature: Overview Toggle Section Then I see the "Collapse All Sections" link And all sections are expanded - @skip-phantom + # Skipped because Ubuntu ChromeDriver hangs on alert + @skip Scenario: Collapse link is not removed after last section of a course is deleted Given I have a course with 1 section And I navigate to the course overview page diff --git a/cms/djangoapps/contentstore/features/subsection.feature b/cms/djangoapps/contentstore/features/subsection.feature index 28285bf8a1..8bb12467ff 100644 --- a/cms/djangoapps/contentstore/features/subsection.feature +++ b/cms/djangoapps/contentstore/features/subsection.feature @@ -3,14 +3,12 @@ Feature: Create Subsection As a course author I want to create and edit subsections - @skip-phantom Scenario: Add a new subsection to a section Given I have opened a new course section in Studio When I click the New Subsection link And I enter the subsection name and click save Then I see my subsection on the Courseware page - @skip-phantom Scenario: Add a new subsection (with a name containing a quote) to a section (bug #216) Given I have opened a new course section in Studio When I click the New Subsection link @@ -27,7 +25,6 @@ Feature: Create Subsection And I reload the page Then I see it marked as Homework - @skip-phantom Scenario: Set a due date in a different year (bug #256) Given I have opened a new subsection in Studio And I have set a release date and due date in different years @@ -35,7 +32,8 @@ Feature: Create Subsection And I reload the page Then I see the correct dates - @skip-phantom + # Skipped because Ubuntu ChromeDriver hangs on alert + @skip Scenario: Delete a subsection Given I have opened a new course section in Studio And I have added a new subsection diff --git a/cms/djangoapps/contentstore/features/subsection.py b/cms/djangoapps/contentstore/features/subsection.py index f9e5b52bb2..edc8b17168 100644 --- a/cms/djangoapps/contentstore/features/subsection.py +++ b/cms/djangoapps/contentstore/features/subsection.py @@ -63,14 +63,6 @@ def test_have_set_dates_in_different_years(step): set_date_and_time('input#due_date', '01/02/2012', 'input#due_time', '04:00') -@step('I see the correct dates$') -def i_see_the_correct_dates(step): - assert_equal('12/25/2011', world.css_find('input#start_date').first.value) - assert_equal('03:00', world.css_find('input#start_time').first.value) - assert_equal('01/02/2012', world.css_find('input#due_date').first.value) - assert_equal('04:00', world.css_find('input#due_time').first.value) - - @step('I mark it as Homework$') def i_mark_it_as_homework(step): world.css_click('a.menu-toggle') @@ -101,8 +93,20 @@ def the_subsection_does_not_exist(step): assert world.browser.is_element_not_present_by_css(css) +@step('I see the correct dates$') +def i_see_the_correct_dates(step): + assert_equal('12/25/2011', get_date('input#start_date')) + assert_equal('03:00', get_date('input#start_time')) + assert_equal('01/02/2012', get_date('input#due_date')) + assert_equal('04:00', get_date('input#due_time')) + + ############ HELPER METHODS ################### +def get_date(css): + return world.css_find(css).first.value.strip() + + def save_subsection_name(name): name_css = 'input.new-subsection-name-input' save_css = 'input.new-subsection-name-save' diff --git a/cms/djangoapps/contentstore/management/commands/update_templates.py b/cms/djangoapps/contentstore/management/commands/update_templates.py index e94fee64b8..36348314b9 100644 --- a/cms/djangoapps/contentstore/management/commands/update_templates.py +++ b/cms/djangoapps/contentstore/management/commands/update_templates.py @@ -1,4 +1,5 @@ from xmodule.templates import update_templates +from xmodule.modulestore.django import modulestore from django.core.management.base import BaseCommand @@ -6,4 +7,4 @@ class Command(BaseCommand): help = 'Imports and updates the Studio component templates from the code pack and put in the DB' def handle(self, *args, **options): - update_templates() + update_templates(modulestore('direct')) diff --git a/cms/djangoapps/contentstore/module_info_model.py b/cms/djangoapps/contentstore/module_info_model.py index 91f722a699..f7d1bbd8fe 100644 --- a/cms/djangoapps/contentstore/module_info_model.py +++ b/cms/djangoapps/contentstore/module_info_model.py @@ -75,11 +75,7 @@ def set_module_info(store, location, post_data): # IMPORTANT NOTE: if the client passed pack 'null' (None) for a piece of metadata that means 'remove it' for metadata_key, value in posted_metadata.items(): - # let's strip out any metadata fields from the postback which have been identified as system metadata - # and therefore should not be user-editable, so we should accept them back from the client - if metadata_key in module.system_metadata_fields: - del posted_metadata[metadata_key] - elif posted_metadata[metadata_key] is None: + if posted_metadata[metadata_key] is None: # remove both from passed in collection as well as the collection read in from the modulestore if metadata_key in module._model_data: del module._model_data[metadata_key] diff --git a/cms/djangoapps/contentstore/tests/test_contentstore.py b/cms/djangoapps/contentstore/tests/test_contentstore.py index 07b7032e60..0aec61729c 100644 --- a/cms/djangoapps/contentstore/tests/test_contentstore.py +++ b/cms/djangoapps/contentstore/tests/test_contentstore.py @@ -220,6 +220,14 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): num_drafts = self._get_draft_counts(course) self.assertEqual(num_drafts, 1) + def test_import_textbook_as_content_element(self): + import_from_xml(modulestore(), 'common/test/data/', ['full']) + + module_store = modulestore('direct') + course = module_store.get_item(Location(['i4x', 'edX', 'full', 'course', '6.002_Spring_2012', None])) + + self.assertGreater(len(course.textbooks), 0) + def test_static_tab_reordering(self): import_from_xml(modulestore(), 'common/test/data/', ['full']) @@ -293,7 +301,6 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): # make sure the parent no longer points to the child object which was deleted self.assertFalse(sequential.location.url() in chapter.children) - def test_about_overrides(self): ''' This test case verifies that a course can use specialized override for about data, e.g. /about/Fall_2012/effort.html @@ -490,6 +497,11 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): self.assertTrue(getattr(test_private_vertical, 'is_draft', False)) + # make sure the textbook survived the export/import + course = module_store.get_item(Location(['i4x', 'edX', 'full', 'course', '6.002_Spring_2012', None])) + + self.assertGreater(len(course.textbooks), 0) + shutil.rmtree(root_dir) def test_course_handouts_rewrites(self): @@ -634,7 +646,7 @@ class ContentStoreTest(ModuleStoreTestCase): resp = self.client.get(reverse('index')) self.assertContains( resp, - '

My Courses

', + '

My Courses

', status_code=200, html=True ) @@ -937,7 +949,7 @@ class TemplateTestCase(ModuleStoreTestCase): self.assertIsNotNone(verify_create) # now run cleanup - update_templates() + update_templates(modulestore('direct')) # now try to find dangling template, it should not be in DB any longer asserted = False diff --git a/cms/djangoapps/contentstore/tests/test_course_updates.py b/cms/djangoapps/contentstore/tests/test_course_updates.py index 80d4f0bbc2..ae14555b32 100644 --- a/cms/djangoapps/contentstore/tests/test_course_updates.py +++ b/cms/djangoapps/contentstore/tests/test_course_updates.py @@ -10,9 +10,9 @@ class CourseUpdateTest(CourseTestCase): '''Go through each interface and ensure it works.''' # first get the update to force the creation url = reverse('course_info', - kwargs={'org': self.course_location.org, - 'course': self.course_location.course, - 'name': self.course_location.name}) + kwargs={'org': self.course_location.org, + 'course': self.course_location.course, + 'name': self.course_location.name}) self.client.get(url) init_content = ' \ No newline at end of file diff --git a/common/test/data/word_cloud/chapter/Staff.xml b/common/test/data/word_cloud/chapter/Staff.xml new file mode 100644 index 0000000000..e1d5216f6d --- /dev/null +++ b/common/test/data/word_cloud/chapter/Staff.xml @@ -0,0 +1,3 @@ + + + diff --git a/common/test/data/word_cloud/course.xml b/common/test/data/word_cloud/course.xml new file mode 100644 index 0000000000..1b97a5a714 --- /dev/null +++ b/common/test/data/word_cloud/course.xml @@ -0,0 +1,2 @@ + + diff --git a/common/test/data/word_cloud/course/2013_Spring.xml b/common/test/data/word_cloud/course/2013_Spring.xml new file mode 100644 index 0000000000..cb6e7c1217 --- /dev/null +++ b/common/test/data/word_cloud/course/2013_Spring.xml @@ -0,0 +1,6 @@ + + + + + diff --git a/common/test/data/word_cloud/creating_course.xml b/common/test/data/word_cloud/creating_course.xml new file mode 100644 index 0000000000..4c90f1c2ec --- /dev/null +++ b/common/test/data/word_cloud/creating_course.xml @@ -0,0 +1,8 @@ + diff --git a/common/test/data/word_cloud/info/2013_Spring/handouts.html b/common/test/data/word_cloud/info/2013_Spring/handouts.html new file mode 100644 index 0000000000..35f2c89474 --- /dev/null +++ b/common/test/data/word_cloud/info/2013_Spring/handouts.html @@ -0,0 +1,3 @@ +
    +
  1. A list of course handouts, or an empty file if there are none.
    2. +
diff --git a/common/test/data/word_cloud/info/2013_Spring/updates.html b/common/test/data/word_cloud/info/2013_Spring/updates.html new file mode 100644 index 0000000000..9744c1699d --- /dev/null +++ b/common/test/data/word_cloud/info/2013_Spring/updates.html @@ -0,0 +1,10 @@ + +
    + +

  1. December 9

    +
    +

    Announcement text

    +
    +
    2. + +
diff --git a/common/test/data/word_cloud/policies/2013_Spring/policy.json b/common/test/data/word_cloud/policies/2013_Spring/policy.json new file mode 100644 index 0000000000..e2a204815c --- /dev/null +++ b/common/test/data/word_cloud/policies/2013_Spring/policy.json @@ -0,0 +1,8 @@ +{ + "course/2013_Spring": { + "start": "2099-01-01T00:00", + "advertised_start" : "Spring 2013", + "display_name": "Justice" + } + +} diff --git a/common/test/data/word_cloud/roots/2013_Spring.xml b/common/test/data/word_cloud/roots/2013_Spring.xml new file mode 100644 index 0000000000..1b97a5a714 --- /dev/null +++ b/common/test/data/word_cloud/roots/2013_Spring.xml @@ -0,0 +1,2 @@ + + diff --git a/common/test/data/word_cloud/sequential/Problem_Demos.xml b/common/test/data/word_cloud/sequential/Problem_Demos.xml new file mode 100644 index 0000000000..21568128a4 --- /dev/null +++ b/common/test/data/word_cloud/sequential/Problem_Demos.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/common/test/data/word_cloud/static/README b/common/test/data/word_cloud/static/README new file mode 100644 index 0000000000..e22f378b5e --- /dev/null +++ b/common/test/data/word_cloud/static/README @@ -0,0 +1,5 @@ +Images, handouts, and other statically-served content should go ONLY +in this directory. + +Images for the front page should go in static/images. The frontpage +banner MUST be named course_image.jpg \ No newline at end of file diff --git a/__init__.py b/common/test/data/word_cloud/word_cloud/cloud.xml similarity index 100% rename from __init__.py rename to common/test/data/word_cloud/word_cloud/cloud.xml diff --git a/conf/locale/config b/conf/locale/config index 2d01e1ea43..58f8da0513 100644 --- a/conf/locale/config +++ b/conf/locale/config @@ -1 +1,4 @@ -{"locales" : ["en"]} +{ + "locales" : ["en", "es"], + "dummy-locale" : "fr" +} diff --git a/conf/locale/en/LC_MESSAGES/messages.po b/conf/locale/en/LC_MESSAGES/messages.po index 1bb8bf6d7f..e5961753c5 100644 --- a/conf/locale/en/LC_MESSAGES/messages.po +++ b/conf/locale/en/LC_MESSAGES/messages.po @@ -1 +1,20 @@ +# edX translation file +# Copyright (C) 2013 edX +# This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE. +# +msgid "" +msgstr "" +"Project-Id-Version: EdX Studio\n" +"Report-Msgid-Bugs-To: translation_team@edx.org\n" +"POT-Creation-Date: 2013-05-02 13:13-0400\n" +"PO-Revision-Date: 2013-05-02 13:27-0400\n" +"Last-Translator: \n" +"Language-Team: translation team \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Language: en\n" + # empty +msgid "This is a key string." +msgstr "" diff --git a/distribute-0.6.32.tar.gz b/distribute-0.6.32.tar.gz deleted file mode 100644 index 2438db60fa..0000000000 Binary files a/distribute-0.6.32.tar.gz and /dev/null differ diff --git a/distribute-0.6.34.tar.gz b/distribute-0.6.34.tar.gz deleted file mode 100644 index 4e91b3af62..0000000000 Binary files a/distribute-0.6.34.tar.gz and /dev/null differ diff --git a/doc/development.md b/doc/development.md index 95cc32329c..c99e99f906 100644 --- a/doc/development.md +++ b/doc/development.md @@ -31,6 +31,14 @@ Check out the course data directories that you want to work with into the rake resetdb +## Installing + +To create your development environment, run the shell script in the root of +the repo: + + scripts/create-dev-env.sh + + ## Starting development servers Both the LMS and Studio can be started using the following shortcut tasks diff --git a/doc/public/course_data_formats/word_cloud/word_cloud.png b/doc/public/course_data_formats/word_cloud/word_cloud.png new file mode 100644 index 0000000000..07b7292b5e Binary files /dev/null and b/doc/public/course_data_formats/word_cloud/word_cloud.png differ diff --git a/doc/public/course_data_formats/word_cloud/word_cloud.rst b/doc/public/course_data_formats/word_cloud/word_cloud.rst new file mode 100644 index 0000000000..5c3d31e149 --- /dev/null +++ b/doc/public/course_data_formats/word_cloud/word_cloud.rst @@ -0,0 +1,59 @@ +********************************************** +Xml format of "Word Cloud" module [xmodule] +********************************************** + +.. module:: word_cloud + +Format description +================== + +The main tag of Word Cloud module input is: + +.. code-block:: xml + + + +The following attributes can be specified for this tag:: + + [display_name| AUTOGENERATE] – Display name of xmodule. When this attribute is not defined - display name autogenerate with some hash. + [num_inputs| 5] – Number of inputs. + [num_top_words| 250] – Number of max words, which will be displayed. + [display_student_percents| True] – Display usage percents for each word on the same line together with words. + +.. note:: + + Percent is shown always when mouse over the word in cloud. + +.. note:: + + Possible answer for boolean type attributes: + True – "True", "true", "T", "t", "1" + False – "False", "false", "F", "f", "0" + +.. note:: + + If you want to use the same word cloud (the same storage of words), you must use the same display_name value. + + +Code Example +============ + +Examples of word_cloud without all attributes (all attributes get by default) +----------------------------------------------------------------------------- + +.. code-block:: xml + + + +Examples of poll with all attributes +------------------------------------ + +.. code-block:: xml + + + +Screenshots +=========== + +.. image:: word_cloud.png + :width: 50% diff --git a/doc/public/index.rst b/doc/public/index.rst index ee681a822e..064b3ff443 100644 --- a/doc/public/index.rst +++ b/doc/public/index.rst @@ -26,6 +26,7 @@ Specific Problem Types course_data_formats/graphical_slider_tool/graphical_slider_tool.rst course_data_formats/poll_module/poll_module.rst course_data_formats/conditional_module/conditional_module.rst + course_data_formats/word_cloud/word_cloud.rst course_data_formats/custom_response.rst diff --git a/doc/testing.md b/doc/testing.md index 84175fee3d..d6c7b7ee86 100644 --- a/doc/testing.md +++ b/doc/testing.md @@ -161,36 +161,36 @@ try running `bundle install` to install the required ruby gems. We use [Lettuce](http://lettuce.it/) for acceptance testing. Most of our tests use [Splinter](http://splinter.cobrateam.info/) to simulate UI browser interactions. Splinter, in turn, -uses [Selenium](http://docs.seleniumhq.org/) to control the browser. +uses [Selenium](http://docs.seleniumhq.org/) to control the Chrome browser. **Prerequisite**: You must have [ChromeDriver](https://code.google.com/p/selenium/wiki/ChromeDriver) -installed to run the tests in Chrome. +installed to run the tests in Chrome. The tests are confirmed to run +with Chrome (not Chromium) version 26.0.0.1410.63 with ChromeDriver +version r195636. -Before running the tests, you need to set up the test database: +To run all the acceptance tests: - rm ../db/test_mitx.db - rake django-admin[syncdb,lms,acceptance,--noinput] - rake django-admin[migrate,lms,acceptance,--noinput] - -To run the acceptance tests: - -1. Start the Django server locally using the settings in **acceptance.py**: - - rake lms[acceptance] - -2. In another shell, run the tests: - - django-admin.py harvest --no-server --settings=lms.envs.acceptance --pythonpath=. lms/djangoapps/portal/features/ + rake test_acceptance_lms + rake test_acceptance_cms To test only a specific feature: - django-admin.py harvest --no-server --settings=lms.envs.acceptance --pythonpath=. lms/djangoapps/courseware/features/high-level-tabs.feature + rake test_acceptance_lms[lms/djangoapps/courseware/features/problems.feature] + +To start the debugger on failure, add the `--pdb` option: + + rake test_acceptance_lms["lms/djangoapps/courseware/features/problems.feature --pdb"] + +To run tests faster by not collecting static files, you can use +`rake fasttest_acceptance_lms` and `rake fasttest_acceptance_cms`. + **Troubleshooting**: If you get an error message that says something about harvest not being a command, you probably are missing a requirement. Try running: pip install -r requirements.txt +**Note**: The acceptance tests can *not* currently run in parallel. ## Viewing Test Coverage diff --git a/docs/source/xmodule.rst b/docs/source/xmodule.rst index 45caa82c30..d68ab779f6 100644 --- a/docs/source/xmodule.rst +++ b/docs/source/xmodule.rst @@ -165,6 +165,13 @@ Video :members: :show-inheritance: +Word Cloud +========== + +.. automodule:: xmodule.word_cloud_module + :members: + :show-inheritance: + X = diff --git a/fixtures/anonymize_fixtures.py b/fixtures/anonymize_fixtures.py deleted file mode 100755 index ba62652de5..0000000000 --- a/fixtures/anonymize_fixtures.py +++ /dev/null @@ -1,98 +0,0 @@ -#! /usr/bin/env python - -import sys -import json -import random -import copy -from collections import defaultdict -from argparse import ArgumentParser, FileType -from datetime import datetime - -def generate_user(user_number): - return { - "pk": user_number, - "model": "auth.user", - "fields": { - "status": "w", - "last_name": "Last", - "gold": 0, - "is_staff": False, - "user_permissions": [], - "interesting_tags": "", - "email_key": None, - "date_joined": "2012-04-26 11:36:39", - "first_name": "", - "email_isvalid": False, - "avatar_type": "n", - "website": "", - "is_superuser": False, - "date_of_birth": None, - "last_login": "2012-04-26 11:36:48", - "location": "", - "new_response_count": 0, - "email": "user{num}@example.com".format(num=user_number), - "username": "user{num}".format(num=user_number), - "is_active": True, - "consecutive_days_visit_count": 0, - "email_tag_filter_strategy": 1, - "groups": [], - "password": "sha1$90e6f$562a1d783a0c47ce06ebf96b8c58123a0671bbf0", - "silver": 0, - "bronze": 0, - "questions_per_page": 10, - "about": "", - "show_country": True, - "country": "", - "display_tag_filter_strategy": 0, - "seen_response_count": 0, - "real_name": "", - "ignored_tags": "", - "reputation": 1, - "gravatar": "366d981a10116969c568a18ee090f44c", - "last_seen": "2012-04-26 11:36:39" - } - } - - -def parse_args(args=sys.argv[1:]): - parser = ArgumentParser() - parser.add_argument('-d', '--data', type=FileType('r'), default=sys.stdin) - parser.add_argument('-o', '--output', type=FileType('w'), default=sys.stdout) - parser.add_argument('count', type=int) - return parser.parse_args(args) - - -def main(args=sys.argv[1:]): - args = parse_args(args) - - data = json.load(args.data) - unique_students = set(entry['fields']['student'] for entry in data) - if args.count > len(unique_students) * 0.1: - raise Exception("Can't be sufficiently anonymous selecting {count} of {unique} students".format( - count=args.count, unique=len(unique_students))) - - by_problems = defaultdict(list) - for entry in data: - by_problems[entry['fields']['module_id']].append(entry) - - out_data = [] - out_pk = 1 - for name, answers in by_problems.items(): - for student_id in xrange(args.count): - sample = random.choice(answers) - data = copy.deepcopy(sample) - data["fields"]["student"] = student_id + 1 - data["fields"]["created"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - data["fields"]["modified"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - data["pk"] = out_pk - out_pk += 1 - out_data.append(data) - - for student_id in xrange(args.count): - out_data.append(generate_user(student_id)) - - json.dump(out_data, args.output, indent=2) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/fixtures/pm.json b/fixtures/pm.json deleted file mode 100644 index 5ecb839093..0000000000 --- a/fixtures/pm.json +++ /dev/null @@ -1 +0,0 @@ -[{"pk": 1, "model": "user.userprofile", "fields": {"name": "pm", "language": "pm", "courseware": "course.xml", "meta": "", "location": "pm", "user": 1}}, {"pk": 1, "model": "auth.user", "fields": {"status": "w", "last_name": "", "gold": 0, "is_staff": true, "user_permissions": [], "interesting_tags": "", "email_key": null, "date_joined": "2012-01-23 17:03:54", "first_name": "", "email_isvalid": false, "avatar_type": "n", "website": "", "is_superuser": true, "date_of_birth": null, "last_login": "2012-01-23 17:04:16", "location": "", "new_response_count": 0, "email": "pmitros@csail.mit.edu", "username": "pm", "is_active": true, "consecutive_days_visit_count": 0, "email_tag_filter_strategy": 1, "groups": [], "password": "sha1$a3e96$dbabbd114f0da01bce2cc2adcafa2ca651c7ae0a", "silver": 0, "bronze": 0, "questions_per_page": 10, "about": "", "show_country": false, "country": "", "display_tag_filter_strategy": 0, "seen_response_count": 0, "real_name": "", "ignored_tags": "", "reputation": 1, "gravatar": "7a591afd0cc7972fdbe5e12e26af352a", "last_seen": "2012-01-23 17:04:41"}}, {"pk": 1, "model": "user.userprofile", "fields": {"name": "pm", "language": "pm", "courseware": "course.xml", "meta": "", "location": "pm", "user": 1}}, {"pk": 1, "model": "auth.user", "fields": {"status": "w", "last_name": "", "gold": 0, "is_staff": true, "user_permissions": [], "interesting_tags": "", "email_key": null, "date_joined": "2012-01-23 17:03:54", "first_name": "", "email_isvalid": false, "avatar_type": "n", "website": "", "is_superuser": true, "date_of_birth": null, "last_login": "2012-01-23 17:04:16", "location": "", "new_response_count": 0, "email": "pmitros@csail.mit.edu", "username": "pm", "is_active": true, "consecutive_days_visit_count": 0, "email_tag_filter_strategy": 1, "groups": [], "password": "sha1$a3e96$dbabbd114f0da01bce2cc2adcafa2ca651c7ae0a", "silver": 0, "bronze": 0, "questions_per_page": 10, "about": "", "show_country": false, "country": "", "display_tag_filter_strategy": 0, "seen_response_count": 0, "real_name": "", "ignored_tags": "", "reputation": 1, "gravatar": "7a591afd0cc7972fdbe5e12e26af352a", "last_seen": "2012-01-23 17:04:41"}}] \ No newline at end of file diff --git a/i18n/config.py b/i18n/config.py new file mode 100644 index 0000000000..4f246ed942 --- /dev/null +++ b/i18n/config.py @@ -0,0 +1,77 @@ +import os, json +from path import path + +# BASE_DIR is the working directory to execute django-admin commands from. +# Typically this should be the 'mitx' directory. +BASE_DIR = path(__file__).abspath().dirname().joinpath('..').normpath() + +# LOCALE_DIR contains the locale files. +# Typically this should be 'mitx/conf/locale' +LOCALE_DIR = BASE_DIR.joinpath('conf', 'locale') + +class Configuration: + """ + # Reads localization configuration in json format + + """ + _source_locale = 'en' + + def __init__(self, filename): + self._filename = filename + self._config = self.read_config(filename) + + def read_config(self, filename): + """ + Returns data found in config file (as dict), or raises exception if file not found + """ + if not os.path.exists(filename): + raise Exception("Configuration file cannot be found: %s" % filename) + with open(filename) as stream: + return json.load(stream) + + @property + def locales(self): + """ + Returns a list of locales declared in the configuration file, + e.g. ['en', 'fr', 'es'] + Each locale is a string. + """ + return self._config['locales'] + + @property + def source_locale(self): + """ + Returns source language. + Source language is English. + """ + return self._source_locale + + @property + def dummy_locale(self): + """ + Returns a locale to use for the dummy text, e.g. 'fr'. + Throws exception if no dummy-locale is declared. + The locale is a string. + """ + dummy = self._config.get('dummy-locale', None) + if not dummy: + raise Exception('Could not read dummy-locale from configuration file.') + return dummy + + def get_messages_dir(self, locale): + """ + Returns the name of the directory holding the po files for locale. + Example: mitx/conf/locale/fr/LC_MESSAGES + """ + return LOCALE_DIR.joinpath(locale, 'LC_MESSAGES') + + @property + def source_messages_dir(self): + """ + Returns the name of the directory holding the source-language po files (English). + Example: mitx/conf/locale/en/LC_MESSAGES + """ + return self.get_messages_dir(self.source_locale) + + +CONFIGURATION = Configuration(LOCALE_DIR.joinpath('config').normpath()) diff --git a/i18n/execute.py b/i18n/execute.py index 3c3416b65d..8e7f0f52de 100644 --- a/i18n/execute.py +++ b/i18n/execute.py @@ -1,69 +1,30 @@ -import os, subprocess, logging, json +import os, subprocess, logging -def init_module(): - """ - Initializes module parameters - """ - global BASE_DIR, LOCALE_DIR, CONFIG_FILENAME, SOURCE_MSGS_DIR, SOURCE_LOCALE, LOG +from config import CONFIGURATION, BASE_DIR - # BASE_DIR is the working directory to execute django-admin commands from. - # Typically this should be the 'mitx' directory. - BASE_DIR = os.path.normpath(os.path.dirname(os.path.abspath(__file__))+'/..') +LOG = logging.getLogger(__name__) - # Source language is English - SOURCE_LOCALE = 'en' - - # LOCALE_DIR contains the locale files. - # Typically this should be 'mitx/conf/locale' - LOCALE_DIR = BASE_DIR + '/conf/locale' - - # CONFIG_FILENAME contains localization configuration in json format - CONFIG_FILENAME = LOCALE_DIR + '/config' - - # SOURCE_MSGS_DIR contains the English po files. - SOURCE_MSGS_DIR = messages_dir(SOURCE_LOCALE) - - # Default logger. - LOG = get_logger() - - -def messages_dir(locale): - """ - Returns the name of the directory holding the po files for locale. - Example: mitx/conf/locale/en/LC_MESSAGES - """ - return os.path.join(LOCALE_DIR, locale, 'LC_MESSAGES') - -def get_logger(): - """Returns a default logger""" - log = logging.getLogger(__name__) - log.setLevel(logging.INFO) - log_handler = logging.StreamHandler() - log_handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(message)s')) - log.addHandler(log_handler) - return log - -# Run this after defining messages_dir and get_logger, because it depends on these. -init_module() - -def execute (command, working_directory=BASE_DIR, log=LOG): +def execute(command, working_directory=BASE_DIR): """ Executes shell command in a given working_directory. Command is a string to pass to the shell. - Output is logged to log. + Output is ignored. """ - log.info(command) + LOG.info(command) subprocess.call(command.split(' '), cwd=working_directory) - -def get_config(): - """Returns data found in config file, or returns None if file not found""" - config_path = os.path.abspath(CONFIG_FILENAME) - if not os.path.exists(config_path): - log.warn("Configuration file cannot be found: %s" % \ - os.path.relpath(config_path, BASE_DIR)) - return None - with open(config_path) as stream: - return json.load(stream) + + +def call(command, working_directory=BASE_DIR): + """ + Executes shell command in a given working_directory. + Command is a string to pass to the shell. + Returns a tuple of two strings: (stdout, stderr) + + """ + LOG.info(command) + p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=working_directory) + out, err = p.communicate() + return (out, err) def create_dir_if_necessary(pathname): dirname = os.path.dirname(pathname) @@ -71,16 +32,16 @@ def create_dir_if_necessary(pathname): os.makedirs(dirname) -def remove_file(filename, log=LOG, verbose=True): +def remove_file(filename, verbose=True): """ Attempt to delete filename. + log is boolean. If true, removal is logged. Log a warning if file does not exist. Logging filenames are releative to BASE_DIR to cut down on noise in output. """ if verbose: - log.info('Deleting file %s' % os.path.relpath(filename, BASE_DIR)) + LOG.info('Deleting file %s' % os.path.relpath(filename, BASE_DIR)) if not os.path.exists(filename): - log.warn("File does not exist: %s" % os.path.relpath(filename, BASE_DIR)) + LOG.warn("File does not exist: %s" % os.path.relpath(filename, BASE_DIR)) else: os.remove(filename) - diff --git a/i18n/extract.py b/i18n/extract.py index c6fedd3bfa..c28c3868e2 100755 --- a/i18n/extract.py +++ b/i18n/extract.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python """ See https://edx-wiki.atlassian.net/wiki/display/ENG/PO+File+workflow @@ -15,28 +15,35 @@ See https://edx-wiki.atlassian.net/wiki/display/ENG/PO+File+workflow """ -import os +import os, sys, logging from datetime import datetime from polib import pofile -from execute import execute, create_dir_if_necessary, remove_file, \ - BASE_DIR, LOCALE_DIR, SOURCE_MSGS_DIR, LOG +from config import BASE_DIR, LOCALE_DIR, CONFIGURATION +from execute import execute, create_dir_if_necessary, remove_file # BABEL_CONFIG contains declarations for Babel to extract strings from mako template files # Use relpath to reduce noise in logs -BABEL_CONFIG = os.path.relpath(LOCALE_DIR + '/babel.cfg', BASE_DIR) +BABEL_CONFIG = BASE_DIR.relpathto(LOCALE_DIR.joinpath('babel.cfg')) # Strings from mako template files are written to BABEL_OUT # Use relpath to reduce noise in logs -BABEL_OUT = os.path.relpath(SOURCE_MSGS_DIR + '/mako.po', BASE_DIR) +BABEL_OUT = BASE_DIR.relpathto(CONFIGURATION.source_messages_dir.joinpath('mako.po')) +SOURCE_WARN = 'This English source file is machine-generated. Do not check it into github' + +LOG = logging.getLogger(__name__) def main (): + logging.basicConfig(stream=sys.stdout, level=logging.INFO) create_dir_if_necessary(LOCALE_DIR) - generated_files = ('django-partial.po', 'djangojs.po', 'mako.po') + source_msgs_dir = CONFIGURATION.source_messages_dir + remove_file(source_msgs_dir.joinpath('django.po')) + generated_files = ('django-partial.po', 'djangojs.po', 'mako.po') for filename in generated_files: - remove_file(os.path.join(SOURCE_MSGS_DIR, filename)) + remove_file(source_msgs_dir.joinpath(filename)) + # Extract strings from mako templates babel_mako_cmd = 'pybabel extract -F %s -c "TRANSLATORS:" . -o %s' % (BABEL_CONFIG, BABEL_OUT) @@ -52,13 +59,13 @@ def main (): execute(make_django_cmd, working_directory=BASE_DIR) # makemessages creates 'django.po'. This filename is hardcoded. # Rename it to django-partial.po to enable merging into django.po later. - os.rename(os.path.join(SOURCE_MSGS_DIR, 'django.po'), - os.path.join(SOURCE_MSGS_DIR, 'django-partial.po')) + os.rename(source_msgs_dir.joinpath('django.po'), + source_msgs_dir.joinpath('django-partial.po')) execute(make_djangojs_cmd, working_directory=BASE_DIR) for filename in generated_files: LOG.info('Cleaning %s' % filename) - po = pofile(os.path.join(SOURCE_MSGS_DIR, filename)) + po = pofile(source_msgs_dir.joinpath(filename)) # replace default headers with edX headers fix_header(po) # replace default metadata with edX metadata @@ -79,10 +86,11 @@ def fix_header(po): """ Replace default headers with edX headers """ + po.metadata_is_fuzzy = [] # remove [u'fuzzy'] header = po.header fixes = ( - ('SOME DESCRIPTIVE TITLE', 'edX translation file'), - ('Translations template for PROJECT.', 'edX translation file'), + ('SOME DESCRIPTIVE TITLE', 'edX translation file\n' + SOURCE_WARN), + ('Translations template for PROJECT.', 'edX translation file\n' + SOURCE_WARN), ('YEAR', '%s' % datetime.utcnow().year), ('ORGANIZATION', 'edX'), ("THE PACKAGE'S COPYRIGHT HOLDER", "EdX"), @@ -119,10 +127,9 @@ def fix_metadata(po): 'Report-Msgid-Bugs-To': 'translation_team@edx.org', 'Project-Id-Version': '0.1a', 'Language' : 'en', + 'Last-Translator' : '', 'Language-Team': 'translation team ', } - if po.metadata.has_key('Last-Translator'): - del po.metadata['Last-Translator'] po.metadata.update(fixes) def strip_key_strings(po): diff --git a/i18n/generate.py b/i18n/generate.py index ddbaadfa70..65c65c00d6 100755 --- a/i18n/generate.py +++ b/i18n/generate.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python """ See https://edx-wiki.atlassian.net/wiki/display/ENG/PO+File+workflow @@ -13,50 +13,71 @@ languages to generate. """ -import os -from execute import execute, get_config, messages_dir, remove_file, \ - BASE_DIR, LOG, SOURCE_LOCALE +import os, sys, logging +from polib import pofile -def merge(locale, target='django.po'): +from config import BASE_DIR, CONFIGURATION +from execute import execute + +LOG = logging.getLogger(__name__) + +def merge(locale, target='django.po', fail_if_missing=True): """ For the given locale, merge django-partial.po, messages.po, mako.po -> django.po + target is the resulting filename + If fail_if_missing is True, and the files to be merged are missing, + throw an Exception. + If fail_if_missing is False, and the files to be merged are missing, + just return silently. """ LOG.info('Merging locale={0}'.format(locale)) - locale_directory = messages_dir(locale) + locale_directory = CONFIGURATION.get_messages_dir(locale) files_to_merge = ('django-partial.po', 'messages.po', 'mako.po') - validate_files(locale_directory, files_to_merge) + try: + validate_files(locale_directory, files_to_merge) + except Exception, e: + if not fail_if_missing: + return + raise e # merged file is merged.po merge_cmd = 'msgcat -o merged.po ' + ' '.join(files_to_merge) execute(merge_cmd, working_directory=locale_directory) + # clean up redunancies in the metadata + merged_filename = locale_directory.joinpath('merged.po') + clean_metadata(merged_filename) + # rename merged.po -> django.po (default) - merged_filename = os.path.join(locale_directory, 'merged.po') - django_filename = os.path.join(locale_directory, target) + django_filename = locale_directory.joinpath(target) os.rename(merged_filename, django_filename) # can't overwrite file on Windows +def clean_metadata(file): + """ + Clean up redundancies in the metadata caused by merging. + This reads in a PO file and simply saves it back out again. + """ + pofile(file).save() + def validate_files(dir, files_to_merge): """ Asserts that the given files exist. files_to_merge is a list of file names (no directories). - dir is the directory in which the files should appear. + dir is the directory (a path object from path.py) in which the files should appear. raises an Exception if any of the files are not in dir. """ for path in files_to_merge: - pathname = os.path.join(dir, path) - if not os.path.exists(pathname): - raise Exception("File not found: {0}".format(pathname)) + pathname = dir.joinpath(path) + if not pathname.exists(): + raise Exception("I18N: Cannot generate because file not found: {0}".format(pathname)) def main (): - configuration = get_config() - if configuration == None: - LOG.warn('Configuration file not found, using only English.') - locales = (SOURCE_LOCALE,) - else: - locales = configuration['locales'] - for locale in locales: - merge(locale) + logging.basicConfig(stream=sys.stdout, level=logging.INFO) + for locale in CONFIGURATION.locales: + merge(locale) + # Dummy text is not required. Don't raise exception if files are missing. + merge(CONFIGURATION.dummy_locale, fail_if_missing=False) compile_cmd = 'django-admin.py compilemessages' execute(compile_cmd, working_directory=BASE_DIR) diff --git a/i18n/make_dummy.py b/i18n/make_dummy.py index c8dcde861a..6c14edd45a 100755 --- a/i18n/make_dummy.py +++ b/i18n/make_dummy.py @@ -1,7 +1,13 @@ -#!/usr/bin/python +#!/usr/bin/env python # Generate test translation files from human-readable po files. # +# Dummy language is specified in configuration file (see config.py) +# two letter language codes reference: +# see http://www.loc.gov/standards/iso639-2/php/code_list.php +# +# Django will not localize in languages that django itself has not been +# localized for. So we are using a well-known language (default='fr'). # # po files can be generated with this: # django-admin.py makemessages --all --extension html -l en @@ -10,14 +16,15 @@ # # $ ./make_dummy.py # -# $ ./make_dummy.py mitx/conf/locale/en/LC_MESSAGES/django.po +# $ ./make_dummy.py ../conf/locale/en/LC_MESSAGES/django.po # # generates output to -# mitx/conf/locale/vr/LC_MESSAGES/django.po +# mitx/conf/locale/fr/LC_MESSAGES/django.po import os, sys import polib from dummy import Dummy +from config import CONFIGURATION from execute import create_dir_if_necessary def main(file, locale): @@ -41,27 +48,19 @@ def new_filename(original_filename, new_locale): orig_dir = os.path.dirname(original_filename) msgs_dir = os.path.basename(orig_dir) orig_file = os.path.basename(original_filename) - return os.path.join(orig_dir, - '/../..', - new_locale, - msgs_dir, - orig_file) - - -# Dummy language -# two letter language codes reference: -# see http://www.loc.gov/standards/iso639-2/php/code_list.php -# -# Django will not localize in languages that django itself has not been -# localized for. So we are using a well-known language: 'fr'. - -DEFAULT_LOCALE = 'fr' + return os.path.abspath(os.path.join(orig_dir, + '../..', + new_locale, + msgs_dir, + orig_file)) if __name__ == '__main__': + # required arg: file if len(sys.argv)<2: raise Exception("missing file argument") - if len(sys.argv)<2: - locale = DEFAULT_LOCALE + # optional arg: locale + if len(sys.argv)<3: + locale = CONFIGURATION.get_dummy_locale() else: locale = sys.argv[2] main(sys.argv[1], locale) diff --git a/i18n/tests/__init__.py b/i18n/tests/__init__.py index d60515c712..ee6283376e 100644 --- a/i18n/tests/__init__.py +++ b/i18n/tests/__init__.py @@ -1,4 +1,6 @@ +from test_config import TestConfiguration from test_extract import TestExtract from test_generate import TestGenerate from test_converter import TestConverter from test_dummy import TestDummy +import test_validate diff --git a/i18n/tests/test_config.py b/i18n/tests/test_config.py new file mode 100644 index 0000000000..bcec6ac354 --- /dev/null +++ b/i18n/tests/test_config.py @@ -0,0 +1,33 @@ +import os +from unittest import TestCase + +from config import Configuration, LOCALE_DIR, CONFIGURATION + +class TestConfiguration(TestCase): + """ + Tests functionality of i18n/config.py + """ + + def test_config(self): + config_filename = os.path.normpath(os.path.join(LOCALE_DIR, 'config')) + config = Configuration(config_filename) + self.assertEqual(config.source_locale, 'en') + + def test_no_config(self): + config_filename = os.path.normpath(os.path.join(LOCALE_DIR, 'no_such_file')) + with self.assertRaises(Exception): + Configuration(config_filename) + + def test_valid_configuration(self): + """ + Make sure we have a valid configuration file, + and that it contains an 'en' locale. + Also check values of dummy_locale and source_locale. + """ + self.assertIsNotNone(CONFIGURATION) + locales = CONFIGURATION.locales + self.assertIsNotNone(locales) + self.assertIsInstance(locales, list) + self.assertIn('en', locales) + self.assertEqual('fr', CONFIGURATION.dummy_locale) + self.assertEqual('en', CONFIGURATION.source_locale) diff --git a/i18n/tests/test_extract.py b/i18n/tests/test_extract.py index b14ae9872d..7e8b1a9d2b 100644 --- a/i18n/tests/test_extract.py +++ b/i18n/tests/test_extract.py @@ -4,7 +4,7 @@ from nose.plugins.skip import SkipTest from datetime import datetime, timedelta import extract -from execute import SOURCE_MSGS_DIR +from config import CONFIGURATION # Make sure setup runs only once SETUP_HAS_RUN = False @@ -39,7 +39,7 @@ class TestExtract(TestCase): Fails assertion if one of the files doesn't exist. """ for filename in self.generated_files: - path = os.path.join(SOURCE_MSGS_DIR, filename) + path = os.path.join(CONFIGURATION.source_messages_dir, filename) exists = os.path.exists(path) self.assertTrue(exists, msg='Missing file: %s' % filename) if exists: diff --git a/i18n/tests/test_generate.py b/i18n/tests/test_generate.py index fc22988251..468858664f 100644 --- a/i18n/tests/test_generate.py +++ b/i18n/tests/test_generate.py @@ -1,9 +1,10 @@ -import os, string, random +import os, string, random, re +from polib import pofile from unittest import TestCase from datetime import datetime, timedelta import generate -from execute import get_config, messages_dir, SOURCE_MSGS_DIR, SOURCE_LOCALE +from config import CONFIGURATION class TestGenerate(TestCase): """ @@ -12,29 +13,16 @@ class TestGenerate(TestCase): generated_files = ('django-partial.po', 'djangojs.po', 'mako.po') def setUp(self): - self.configuration = get_config() - # Subtract 1 second to help comparisons with file-modify time succeed, # since os.path.getmtime() is not millisecond-accurate self.start_time = datetime.now() - timedelta(seconds=1) - def test_configuration(self): - """ - Make sure we have a valid configuration file, - and that it contains an 'en' locale. - """ - self.assertIsNotNone(self.configuration) - locales = self.configuration['locales'] - self.assertIsNotNone(locales) - self.assertIsInstance(locales, list) - self.assertIn('en', locales) - def test_merge(self): """ Tests merge script on English source files. """ - filename = os.path.join(SOURCE_MSGS_DIR, random_name()) - generate.merge(SOURCE_LOCALE, target=filename) + filename = os.path.join(CONFIGURATION.source_messages_dir, random_name()) + generate.merge(CONFIGURATION.source_locale, target=filename) self.assertTrue(os.path.exists(filename)) os.remove(filename) @@ -47,13 +35,35 @@ class TestGenerate(TestCase): after start of test suite) """ generate.main() - for locale in self.configuration['locales']: - for filename in ('django.mo', 'djangojs.mo'): - path = os.path.join(messages_dir(locale), filename) + for locale in CONFIGURATION.locales: + for filename in ('django', 'djangojs'): + mofile = filename+'.mo' + path = os.path.join(CONFIGURATION.get_messages_dir(locale), mofile) exists = os.path.exists(path) - self.assertTrue(exists, msg='Missing file in locale %s: %s' % (locale, filename)) + self.assertTrue(exists, msg='Missing file in locale %s: %s' % (locale, mofile)) self.assertTrue(datetime.fromtimestamp(os.path.getmtime(path)) >= self.start_time, msg='File not recently modified: %s' % path) + self.assert_merge_headers(locale) + + def assert_merge_headers(self, locale): + """ + This is invoked by test_main to ensure that it runs after + calling generate.main(). + + There should be exactly three merge comment headers + in our merged .po file. This counts them to be sure. + A merge comment looks like this: + # #-#-#-#-# django-partial.po (0.1a) #-#-#-#-# + + """ + path = os.path.join(CONFIGURATION.get_messages_dir(locale), 'django.po') + po = pofile(path) + pattern = re.compile('^#-#-#-#-#', re.M) + match = pattern.findall(po.header) + self.assertEqual(len(match), 3, + msg="Found %s (should be 3) merge comments in the header for %s" % \ + (len(match), path)) + def random_name(size=6): """Returns random filename as string, like test-4BZ81W""" diff --git a/i18n/tests/test_validate.py b/i18n/tests/test_validate.py new file mode 100644 index 0000000000..bef563faea --- /dev/null +++ b/i18n/tests/test_validate.py @@ -0,0 +1,34 @@ +import os, sys, logging +from unittest import TestCase +from nose.plugins.skip import SkipTest + +from config import LOCALE_DIR +from execute import call + +def test_po_files(root=LOCALE_DIR): + """ + This is a generator. It yields all of the .po files under root, and tests each one. + """ + log = logging.getLogger(__name__) + logging.basicConfig(stream=sys.stdout, level=logging.INFO) + + for (dirpath, dirnames, filenames) in os.walk(root): + for name in filenames: + (base, ext) = os.path.splitext(name) + if ext.lower() == '.po': + yield validate_po_file, os.path.join(dirpath, name), log + + +def validate_po_file(filename, log): + """ + Call GNU msgfmt -c on each .po file to validate its format. + Any errors caught by msgfmt are logged to log. + """ + # Skip this test for now because it's very noisy + raise SkipTest() + # Use relative paths to make output less noisy. + rfile = os.path.relpath(filename, LOCALE_DIR) + (out, err) = call(['msgfmt','-c', rfile], working_directory=LOCALE_DIR) + if err != '': + log.warn('\n'+err) + diff --git a/i18n/transifex.py b/i18n/transifex.py new file mode 100755 index 0000000000..ac203f3eea --- /dev/null +++ b/i18n/transifex.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +import os, sys +from polib import pofile +from config import CONFIGURATION +from extract import SOURCE_WARN +from execute import execute + +TRANSIFEX_HEADER = 'Translations in this file have been downloaded from %s' +TRANSIFEX_URL = 'https://www.transifex.com/projects/p/edx-studio/' + +def push(): + execute('tx push -s') + +def pull(): + for locale in CONFIGURATION.locales: + if locale != CONFIGURATION.source_locale: + execute('tx pull -l %s' % locale) + clean_translated_locales() + + +def clean_translated_locales(): + """ + Strips out the warning from all translated po files + about being an English source file. + """ + for locale in CONFIGURATION.locales: + if locale != CONFIGURATION.source_locale: + clean_locale(locale) + +def clean_locale(locale): + """ + Strips out the warning from all of a locale's translated po files + about being an English source file. + Iterates over machine-generated files. + """ + dirname = CONFIGURATION.get_messages_dir(locale) + for filename in ('django-partial.po', 'djangojs.po', 'mako.po'): + clean_file(dirname.joinpath(filename)) + +def clean_file(file): + """ + Strips out the warning from a translated po file about being an English source file. + Replaces warning with a note about coming from Transifex. + """ + po = pofile(file) + if po.header.find(SOURCE_WARN) != -1: + new_header = get_new_header(po) + new = po.header.replace(SOURCE_WARN, new_header) + po.header = new + po.save() + +def get_new_header(po): + team = po.metadata.get('Language-Team', None) + if not team: + return TRANSIFEX_HEADER % TRANSIFEX_URL + else: + return TRANSIFEX_HEADER % team + +if __name__ == '__main__': + if len(sys.argv)<2: + raise Exception("missing argument: push or pull") + arg = sys.argv[1] + if arg == 'push': + push() + elif arg == 'pull': + pull() + else: + raise Exception("unknown argument: (%s)" % arg) + diff --git a/install.txt b/install.txt deleted file mode 100644 index 801036af6b..0000000000 --- a/install.txt +++ /dev/null @@ -1,74 +0,0 @@ -This document describes how to set up the MITx development environment -for both Linux (Ubuntu) and MacOS (OSX Lion). - -There is also a script "create-dev-env.sh" that automates these steps. - -1) Make an mitx_all directory and clone the repos - (download and install git and mercurial if you don't have them already) - - mkdir ~/mitx_all - cd ~/mitx_all - git clone git@github.com:MITx/mitx.git - hg clone ssh://hg-content@gp.mitx.mit.edu/data - -2) Install OSX dependencies (Mac users only) - - a) Install the brew utility if necessary - /usr/bin/ruby -e "$(curl -fsSL https://raw.github.com/mxcl/homebrew/master/Library/Contributions/install_homebrew.rb)" - - b) Install the brew package list - cat ~/mitx_all/mitx/brew-formulas.txt | xargs brew install - - c) Install python pip if necessary - sudo easy_install pip - - d) Install python virtualenv if necessary - sudo pip install virtualenv virtualenvwrapper - - e) Install coffee script - curl http://npmjs.org/install.sh | sh - npm install -g coffee-script - -3) Install Ubuntu dependencies (Linux users only) - - sudo apt-get install curl python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript - - -4) Install rvm, ruby, and libraries - - echo "export rvm_path=$HOME/mitx_all/ruby" > $HOME/.rvmrc - curl -sL get.rvm.io | bash -s stable - source ~/mitx_all/ruby/scripts/rvm - rvm install 1.9.3 - gem install bundler - cd ~/mitx_all/mitx - bundle install - -5) Install python libraries - - source ~/mitx_all/python/bin/activate - cd ~/mitx_all - pip install -r mitx/pre-requirements.txt - pip install -r mitx/requirements.txt - -6) Create log and db dirs - - mkdir ~/mitx_all/log - mkdir ~/mitx_all/db - -7) Start the dev server - - To start using Django you will need - to activate the local Python and Ruby - environment: - - $ source ~/mitx_all/ruby/scripts/rvm - $ source ~/mitx_all/python/bin/activate - - To initialize and start a local instance of Django: - - $ cd ~/mitx_all/mitx - $ django-admin.py syncdb --settings=envs.dev --pythonpath=. - $ django-admin.py migrate --settings=envs.dev --pythonpath=. - $ django-admin.py runserver --settings=envs.dev --pythonpath=. - diff --git a/jenkins/base.sh b/jenkins/base.sh deleted file mode 100644 index fc2595662a..0000000000 --- a/jenkins/base.sh +++ /dev/null @@ -1,12 +0,0 @@ - -function github_status { - gcli status create edx mitx $GIT_COMMIT \ - --params=$1 \ - target_url:$BUILD_URL \ - description:"Build #$BUILD_NUMBER is running" \ - -f csv -} - -function github_mark_failed_on_exit { - trap '[ $? == "0" ] || github_status state:failed' EXIT -} diff --git a/jenkins/test.sh b/jenkins/test.sh index 7475076086..32279fe22f 100755 --- a/jenkins/test.sh +++ b/jenkins/test.sh @@ -3,8 +3,21 @@ set -e set -x +## +## requires >= 1.3.0 of the Jenkins git plugin +## + function github_status { - gcli status create edx mitx $GIT_COMMIT \ + if [[ ! ${GIT_URL} =~ git@github.com:([^/]+)/([^\.]+).git ]]; then + echo "Cannot parse Github org or repo from URL, using defaults." + ORG="edx" + REPO="edx-platform" + else + ORG=${BASH_REMATCH[1]} + REPO=${BASH_REMATCH[2]} + fi + + gcli status create $ORG $REPO $GIT_COMMIT \ --params=$1 \ target_url:$BUILD_URL \ description:"Build #$BUILD_NUMBER $2" \ @@ -27,21 +40,32 @@ git submodule foreach 'git reset --hard HEAD' export PYTHONIOENCODING=UTF-8 GIT_BRANCH=${GIT_BRANCH/HEAD/master} -if [ ! -d /mnt/virtualenvs/"$JOB_NAME" ]; then - mkdir -p /mnt/virtualenvs/"$JOB_NAME" - virtualenv /mnt/virtualenvs/"$JOB_NAME" + +# When running in parallel on jenkins, workspace could be suffixed by @x +# In that case, we want to use a separate virtualenv that matches up with +# workspace +# +# We need to handle both the case of /path/to/workspace +# and /path/to/workspace@2, which is why we use the following substitutions +# +# $WORKSPACE is the absolute path for the workspace +WORKSPACE_SUFFIX=$(expr "$WORKSPACE" : '.*\(@.*\)') || true + +VIRTUALENV_DIR="/mnt/virtualenvs/${JOB_NAME}${WORKSPACE_SUFFIX}" + +if [ ! -d "$VIRTUALENV_DIR" ]; then + mkdir -p "$VIRTUALENV_DIR" + virtualenv "$VIRTUALENV_DIR" fi export PIP_DOWNLOAD_CACHE=/mnt/pip-cache +# Allow django liveserver tests to use a range of ports +export DJANGO_LIVE_TEST_SERVER_ADDRESS=${DJANGO_LIVE_TEST_SERVER_ADDRESS-localhost:8000-9000} + source /mnt/virtualenvs/"$JOB_NAME"/bin/activate -pip install -q -r pre-requirements.txt -yes w | pip install -q -r requirements.txt - -bundle install - -npm install +rake install_prereqs rake clobber rake pep8 > pep8.log || cat pep8.log rake pylint > pylint.log || cat pylint.log @@ -54,7 +78,7 @@ rake test_lms[false] || TESTS_FAILED=1 rake test_common/lib/capa || TESTS_FAILED=1 rake test_common/lib/xmodule || TESTS_FAILED=1 -# Run the jaavascript unit tests +# Run the javascript unit tests rake phantomjs_jasmine_lms || TESTS_FAILED=1 rake phantomjs_jasmine_cms || TESTS_FAILED=1 rake phantomjs_jasmine_common/lib/xmodule || TESTS_FAILED=1 diff --git a/jenkins/test_acceptance.sh b/jenkins/test_acceptance.sh new file mode 100755 index 0000000000..1d11265d08 --- /dev/null +++ b/jenkins/test_acceptance.sh @@ -0,0 +1,39 @@ +#! /bin/bash + +set -e +set -x + +git remote prune origin + +# Reset the submodule, in case it changed +git submodule foreach 'git reset --hard HEAD' + +# Set the IO encoding to UTF-8 so that askbot will start +export PYTHONIOENCODING=UTF-8 + +if [ ! -d /mnt/virtualenvs/"$JOB_NAME" ]; then + mkdir -p /mnt/virtualenvs/"$JOB_NAME" + virtualenv /mnt/virtualenvs/"$JOB_NAME" +fi + +export PIP_DOWNLOAD_CACHE=/mnt/pip-cache + +source /mnt/virtualenvs/"$JOB_NAME"/bin/activate +rake install_prereqs +rake clobber + +TESTS_FAILED=0 + +# Assumes that Xvfb has been started by upstart +# and is capturing display :1 +# The command for this is: +# /usr/bin/Xvfb :1 -screen 0 1024x268x24 +# This allows us to run Chrome without a display +export DISPLAY=:1 + +# Run the lms and cms acceptance tests +# (the -v flag turns off color in the output) +rake test_acceptance_lms["-v 3"] || TESTS_FAILED=1 +rake test_acceptance_cms["-v 3"] || TESTS_FAILED=1 + +[ $TESTS_FAILED == '0' ] diff --git a/lms/djangoapps/courseware/tabs.py b/lms/djangoapps/courseware/tabs.py index 9f9a4e3e96..ea6f2fc556 100644 --- a/lms/djangoapps/courseware/tabs.py +++ b/lms/djangoapps/courseware/tabs.py @@ -294,6 +294,27 @@ def get_course_tabs(user, course, active_page): return tabs +def get_discussion_link(course): + """ + Return the URL for the discussion tab for the given `course`. + + If they have a discussion link specified, use that even if we disable + discussions. Disabling discsussions is mostly a server safety feature at + this point, and we don't need to worry about external sites. Otherwise, + if the course has a discussion tab or uses the default tabs, return the + discussion view URL. Otherwise, return None to indicate the lack of a + discussion tab. + """ + if course.discussion_link: + return course.discussion_link + elif not settings.MITX_FEATURES.get('ENABLE_DISCUSSION_SERVICE'): + return None + elif hasattr(course, 'tabs') and course.tabs and not any([tab['type'] == 'discussion' for tab in course.tabs]): + return None + else: + return reverse('django_comment_client.forum.views.forum_form_discussion', args=[course.id]) + + def get_default_tabs(user, course, active_page): # When calling the various _tab methods, can omit the 'type':'blah' from the @@ -308,15 +329,9 @@ def get_default_tabs(user, course, active_page): tabs.extend(_textbooks({}, user, course, active_page)) - ## If they have a discussion link specified, use that even if we feature - ## flag discussions off. Disabling that is mostly a server safety feature - ## at this point, and we don't need to worry about external sites. - if course.discussion_link: - tabs.append(CourseTab('Discussion', course.discussion_link, active_page == 'discussion')) - elif settings.MITX_FEATURES.get('ENABLE_DISCUSSION_SERVICE'): - link = reverse('django_comment_client.forum.views.forum_form_discussion', - args=[course.id]) - tabs.append(CourseTab('Discussion', link, active_page == 'discussion')) + discussion_link = get_discussion_link(course) + if discussion_link: + tabs.append(CourseTab('Discussion', discussion_link, active_page == 'discussion')) tabs.extend(_wiki({'name': 'Wiki', 'type': 'wiki'}, user, course, active_page)) diff --git a/lms/djangoapps/courseware/tests/test_tabs.py b/lms/djangoapps/courseware/tests/test_tabs.py index 928b9ae0df..04c46a7820 100644 --- a/lms/djangoapps/courseware/tests/test_tabs.py +++ b/lms/djangoapps/courseware/tests/test_tabs.py @@ -1,11 +1,15 @@ from django.test import TestCase from mock import MagicMock +from mock import patch import courseware.tabs as tabs from django.test.utils import override_settings from django.core.urlresolvers import reverse +from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory class ProgressTestCase(TestCase): @@ -257,3 +261,62 @@ class ValidateTabsTestCase(TestCase): self.assertRaises(tabs.InvalidTabsException, tabs.validate_tabs, self.courses[2]) self.assertIsNone(tabs.validate_tabs(self.courses[3])) self.assertRaises(tabs.InvalidTabsException, tabs.validate_tabs, self.courses[4]) + + +@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) +class DiscussionLinkTestCase(ModuleStoreTestCase): + + def setUp(self): + self.tabs_with_discussion = [ + {'type':'courseware'}, + {'type':'course_info'}, + {'type':'discussion'}, + {'type':'textbooks'}, + ] + self.tabs_without_discussion = [ + {'type':'courseware'}, + {'type':'course_info'}, + {'type':'textbooks'}, + ] + + @staticmethod + def _patch_reverse(course): + def patched_reverse(viewname, args): + if viewname == "django_comment_client.forum.views.forum_form_discussion" and args == [course.id]: + return "default_discussion_link" + else: + return None + return patch("courseware.tabs.reverse", patched_reverse) + + @patch.dict("django.conf.settings.MITX_FEATURES", {"ENABLE_DISCUSSION_SERVICE": False}) + def test_explicit_discussion_link(self): + """Test that setting discussion_link overrides everything else""" + course = CourseFactory.create(discussion_link="other_discussion_link", tabs=self.tabs_with_discussion) + self.assertEqual(tabs.get_discussion_link(course), "other_discussion_link") + + @patch.dict("django.conf.settings.MITX_FEATURES", {"ENABLE_DISCUSSION_SERVICE": False}) + def test_discussions_disabled(self): + """Test that other cases return None with discussions disabled""" + for i, t in enumerate([None, self.tabs_with_discussion, self.tabs_without_discussion]): + course = CourseFactory.create(tabs=t, number=str(i)) + self.assertEqual(tabs.get_discussion_link(course), None) + + @patch.dict("django.conf.settings.MITX_FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + def test_no_tabs(self): + """Test a course without tabs configured""" + course = CourseFactory.create(tabs=None) + with self._patch_reverse(course): + self.assertEqual(tabs.get_discussion_link(course), "default_discussion_link") + + @patch.dict("django.conf.settings.MITX_FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + def test_tabs_with_discussion(self): + """Test a course with a discussion tab configured""" + course = CourseFactory.create(tabs=self.tabs_with_discussion) + with self._patch_reverse(course): + self.assertEqual(tabs.get_discussion_link(course), "default_discussion_link") + + @patch.dict("django.conf.settings.MITX_FEATURES", {"ENABLE_DISCUSSION_SERVICE": True}) + def test_tabs_without_discussion(self): + """Test a course with tabs configured but without a discussion tab""" + course = CourseFactory.create(tabs=self.tabs_without_discussion) + self.assertEqual(tabs.get_discussion_link(course), None) diff --git a/lms/djangoapps/courseware/tests/tests.py b/lms/djangoapps/courseware/tests/tests.py index df141fe284..a189160a48 100644 --- a/lms/djangoapps/courseware/tests/tests.py +++ b/lms/djangoapps/courseware/tests/tests.py @@ -1,13 +1,13 @@ ''' Test for lms courseware app ''' - import logging import json import time import random from urlparse import urlsplit, urlunsplit +from uuid import uuid4 from django.contrib.auth.models import User, Group from django.test import TestCase @@ -62,7 +62,7 @@ def mongo_store_config(data_dir): 'default_class': 'xmodule.raw_module.RawDescriptor', 'host': 'localhost', 'db': 'test_xmodule', - 'collection': 'modulestore', + 'collection': 'modulestore_%s' % uuid4().hex, 'fs_root': data_dir, 'render_template': 'mitxmako.shortcuts.render_to_string', } @@ -81,7 +81,7 @@ def draft_mongo_store_config(data_dir): 'default_class': 'xmodule.raw_module.RawDescriptor', 'host': 'localhost', 'db': 'test_xmodule', - 'collection': 'modulestore', + 'collection': 'modulestore_%s' % uuid4().hex, 'fs_root': data_dir, 'render_template': 'mitxmako.shortcuts.render_to_string', } @@ -92,7 +92,7 @@ def draft_mongo_store_config(data_dir): 'default_class': 'xmodule.raw_module.RawDescriptor', 'host': 'localhost', 'db': 'test_xmodule', - 'collection': 'modulestore', + 'collection': 'modulestore_%s' % uuid4().hex, 'fs_root': data_dir, 'render_template': 'mitxmako.shortcuts.render_to_string', } @@ -401,6 +401,14 @@ class TestCoursesLoadTestCase_MongoModulestore(PageLoaderTestCase): import_from_xml(module_store, TEST_DATA_DIR, ['toy']) self.check_random_page_loads(module_store) + def test_full_textbooks_loads(self): + module_store = modulestore() + import_from_xml(module_store, TEST_DATA_DIR, ['full']) + + course = module_store.get_item(Location(['i4x', 'edX', 'full', 'course', '6.002_Spring_2012', None])) + + self.assertGreater(len(course.textbooks), 0) + @override_settings(MODULESTORE=TEST_DATA_XML_MODULESTORE) class TestNavigation(LoginEnrollmentTestCase): @@ -622,8 +630,8 @@ class TestViewAuth(LoginEnrollmentTestCase): urls = reverse_urls(['info', 'progress'], course) urls.extend([ reverse('book', kwargs={'course_id': course.id, - 'book_index': book.title}) - for book in course.textbooks + 'book_index': index}) + for index, book in enumerate(course.textbooks) ]) return urls @@ -634,8 +642,6 @@ class TestViewAuth(LoginEnrollmentTestCase): """ urls = reverse_urls(['about_course'], course) urls.append(reverse('courses')) - # Need separate test for change_enrollment, since it's a POST view - #urls.append(reverse('change_enrollment')) return urls diff --git a/lms/djangoapps/instructor/tests/test_gradebook.py b/lms/djangoapps/instructor/tests/test_gradebook.py index 2de5c18bcd..4b1d22b594 100644 --- a/lms/djangoapps/instructor/tests/test_gradebook.py +++ b/lms/djangoapps/instructor/tests/test_gradebook.py @@ -49,7 +49,6 @@ class TestGradebook(ModuleStoreTestCase): ] for user in self.users: - UserProfileFactory.create(user=user) CourseEnrollmentFactory.create(user=user, course_id=self.course.id) for i in xrange(USER_COUNT-1): @@ -151,4 +150,4 @@ class TestLetterCutoffPolicy(TestGradebook): # User 0 has 0 on Homeworks [1] # User 0 has 0 on the class [1] # One use at the top of the page [1] - self.assertEquals(3, self.response.content.count('grade_None')) \ No newline at end of file + self.assertEquals(3, self.response.content.count('grade_None')) diff --git a/lms/djangoapps/open_ended_grading/open_ended_notifications.py b/lms/djangoapps/open_ended_grading/open_ended_notifications.py index 6d5f2a3eb4..1d6fa22929 100644 --- a/lms/djangoapps/open_ended_grading/open_ended_notifications.py +++ b/lms/djangoapps/open_ended_grading/open_ended_notifications.py @@ -11,6 +11,7 @@ from util.cache import cache import datetime from xmodule.x_module import ModuleSystem from mitxmako.shortcuts import render_to_string +import datetime log = logging.getLogger(__name__) @@ -104,6 +105,25 @@ def peer_grading_notifications(course, user): def combined_notifications(course, user): + """ + Show notifications to a given user for a given course. Get notifications from the cache if possible, + or from the grading controller server if not. + @param course: The course object for which we are getting notifications + @param user: The user object for which we are getting notifications + @return: A dictionary with boolean pending_grading (true if there is pending grading), img_path (for notification + image), and response (actual response from grading controller server). + """ + #Set up return values so that we can return them for error cases + pending_grading = False + img_path = "" + notifications={} + notification_dict = {'pending_grading': pending_grading, 'img_path': img_path, 'response': notifications} + + #We don't want to show anonymous users anything. + if not user.is_authenticated(): + return notification_dict + + #Define a mock modulesystem system = ModuleSystem( ajax_url=None, track_function=None, @@ -112,41 +132,44 @@ def combined_notifications(course, user): replace_urls=None, xblock_model_data= {} ) + #Initialize controller query service using our mock system controller_qs = ControllerQueryService(settings.OPEN_ENDED_GRADING_INTERFACE, system) student_id = unique_id_for_user(user) user_is_staff = has_access(user, course, 'staff') course_id = course.id notification_type = "combined" + #See if we have a stored value in the cache success, notification_dict = get_value_from_cache(student_id, course_id, notification_type) if success: return notification_dict - min_time_to_query = user.last_login + #Get the time of the last login of the user + last_login = user.last_login + + #Find the modules they have seen since they logged in last_module_seen = StudentModule.objects.filter(student=user, course_id=course_id, - modified__gt=min_time_to_query).values('modified').order_by( + modified__gt=last_login).values('modified').order_by( '-modified') last_module_seen_count = last_module_seen.count() if last_module_seen_count > 0: + #The last time they viewed an updated notification (last module seen minus how long notifications are cached) last_time_viewed = last_module_seen[0]['modified'] - datetime.timedelta(seconds=(NOTIFICATION_CACHE_TIME + 60)) else: - last_time_viewed = user.last_login + #If they have not seen any modules since they logged in, then don't refresh + return {'pending_grading': False, 'img_path': img_path, 'response': notifications} - pending_grading = False - - img_path = "" try: + #Get the notifications from the grading controller controller_response = controller_qs.check_combined_notifications(course.id, student_id, user_is_staff, last_time_viewed) - log.debug(controller_response) notifications = json.loads(controller_response) if notifications['success']: if notifications['overall_need_to_check']: pending_grading = True except: #Non catastrophic error, so no real action - notifications = {} #This is a dev_facing_error log.exception( "Problem with getting notifications from controller query service for course {0} user {1}.".format( @@ -157,6 +180,7 @@ def combined_notifications(course, user): notification_dict = {'pending_grading': pending_grading, 'img_path': img_path, 'response': notifications} + #Store the notifications in the cache set_value_in_cache(student_id, course_id, notification_type, notification_dict) return notification_dict diff --git a/lms/djangoapps/open_ended_grading/tests.py b/lms/djangoapps/open_ended_grading/tests.py index 93d27d8e24..ffc02608d5 100644 --- a/lms/djangoapps/open_ended_grading/tests.py +++ b/lms/djangoapps/open_ended_grading/tests.py @@ -84,7 +84,9 @@ class TestStaffGradingService(LoginEnrollmentTestCase): data = {'location': self.location} r = self.check_for_post_code(200, url, data) + d = json.loads(r.content) + self.assertTrue(d['success']) self.assertEquals(d['submission_id'], self.mock_service.cnt) self.assertIsNotNone(d['submission']) @@ -130,6 +132,7 @@ class TestStaffGradingService(LoginEnrollmentTestCase): r = self.check_for_post_code(200, url, data) d = json.loads(r.content) + self.assertTrue(d['success'], str(d)) self.assertIsNotNone(d['problem_list']) @@ -179,7 +182,8 @@ class TestPeerGradingService(LoginEnrollmentTestCase): data = {'location': self.location} r = self.peer_module.get_next_submission(data) - d = json.loads(r) + d = r + self.assertTrue(d['success']) self.assertIsNotNone(d['submission_id']) self.assertIsNotNone(d['prompt']) @@ -213,7 +217,8 @@ class TestPeerGradingService(LoginEnrollmentTestCase): qdict.keys = data.keys r = self.peer_module.save_grade(qdict) - d = json.loads(r) + d = r + self.assertTrue(d['success']) def test_save_grade_missing_keys(self): @@ -225,7 +230,8 @@ class TestPeerGradingService(LoginEnrollmentTestCase): def test_is_calibrated_success(self): data = {'location': self.location} r = self.peer_module.is_student_calibrated(data) - d = json.loads(r) + d = r + self.assertTrue(d['success']) self.assertTrue('calibrated' in d) @@ -239,9 +245,8 @@ class TestPeerGradingService(LoginEnrollmentTestCase): data = {'location': self.location} r = self.peer_module.show_calibration_essay(data) - d = json.loads(r) - log.debug(d) - log.debug(type(d)) + d = r + self.assertTrue(d['success']) self.assertIsNotNone(d['submission_id']) self.assertIsNotNone(d['prompt']) diff --git a/lms/envs/acceptance.py b/lms/envs/acceptance.py index 2c51dda5e6..611c3fdac8 100644 --- a/lms/envs/acceptance.py +++ b/lms/envs/acceptance.py @@ -8,13 +8,17 @@ from .test import * # otherwise the browser will not render the pages correctly DEBUG = True +# Disable warnings for acceptance tests, to make the logs readable +import logging +logging.disable(logging.ERROR) + # Use the mongo store for acceptance tests modulestore_options = { 'default_class': 'xmodule.raw_module.RawDescriptor', 'host': 'localhost', 'db': 'test_xmodule', - 'collection': 'modulestore', - 'fs_root': GITHUB_REPO_ROOT, + 'collection': 'acceptance_modulestore', + 'fs_root': TEST_ROOT / "data", 'render_template': 'mitxmako.shortcuts.render_to_string', } @@ -33,7 +37,7 @@ CONTENTSTORE = { 'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore', 'OPTIONS': { 'host': 'localhost', - 'db': 'test_xcontent', + 'db': 'test_xmodule', } } @@ -43,8 +47,8 @@ CONTENTSTORE = { DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ENV_ROOT / "db" / "test_mitx.db", - 'TEST_NAME': ENV_ROOT / "db" / "test_mitx.db", + 'NAME': TEST_ROOT / "db" / "test_mitx.db", + 'TEST_NAME': TEST_ROOT / "db" / "test_mitx.db", } } diff --git a/lms/envs/aws.py b/lms/envs/aws.py index c7e6c0e50f..74540b7dec 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -64,6 +64,7 @@ CACHES = ENV_TOKENS['CACHES'] DEFAULT_FROM_EMAIL = ENV_TOKENS.get('DEFAULT_FROM_EMAIL', DEFAULT_FROM_EMAIL) DEFAULT_FEEDBACK_EMAIL = ENV_TOKENS.get('DEFAULT_FEEDBACK_EMAIL', DEFAULT_FEEDBACK_EMAIL) ADMINS = ENV_TOKENS.get('ADMINS', ADMINS) +SERVER_EMAIL = ENV_TOKENS.get('SERVER_EMAIL', SERVER_EMAIL) #Timezone overrides TIME_ZONE = ENV_TOKENS.get('TIME_ZONE', TIME_ZONE) @@ -88,6 +89,8 @@ META_UNIVERSITIES = ENV_TOKENS.get('META_UNIVERSITIES', {}) COMMENTS_SERVICE_URL = ENV_TOKENS.get("COMMENTS_SERVICE_URL", '') COMMENTS_SERVICE_KEY = ENV_TOKENS.get("COMMENTS_SERVICE_KEY", '') CERT_QUEUE = ENV_TOKENS.get("CERT_QUEUE", 'test-pull') +ZENDESK_URL = ENV_TOKENS.get("ZENDESK_URL") +FEEDBACK_SUBMISSION_EMAIL = ENV_TOKENS.get("FEEDBACK_SUBMISSION_EMAIL") for name, value in ENV_TOKENS.get("CODE_JAIL", {}).items(): oldvalue = CODE_JAIL.get(name) @@ -133,3 +136,6 @@ DATADOG_API = AUTH_TOKENS.get("DATADOG_API") # Analytics dashboard server ANALYTICS_SERVER_URL = ENV_TOKENS.get("ANALYTICS_SERVER_URL") ANALYTICS_API_KEY = AUTH_TOKENS.get("ANALYTICS_API_KEY", "") + +ZENDESK_USER = AUTH_TOKENS.get("ZENDESK_USER") +ZENDESK_API_KEY = AUTH_TOKENS.get("ZENDESK_API_KEY") diff --git a/lms/envs/common.py b/lms/envs/common.py index c9c53fd034..bb6388ff93 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -92,6 +92,9 @@ MITX_FEATURES = { # Staff Debug tool. 'ENABLE_STUDENT_HISTORY_VIEW': True, + # Provide a UI to allow users to submit feedback from the LMS + 'ENABLE_FEEDBACK_SUBMISSION': False, + # Turn on a page that lets staff enter Python code to be run in the # sandbox, for testing whether it's enabled properly. 'ENABLE_DEBUG_RUN_PYTHON': False, @@ -124,9 +127,7 @@ sys.path.append(COMMON_ROOT / 'lib') # For Node.js -system_node_path = os.environ.get("NODE_PATH", None) -if system_node_path is None: - system_node_path = "/usr/local/lib/node_modules" +system_node_path = os.environ.get("NODE_PATH", REPO_ROOT / 'node_modules') node_paths = [COMMON_ROOT / "static/js/vendor", COMMON_ROOT / "static/coffee/src", @@ -291,6 +292,7 @@ IGNORABLE_404_ENDS = ('favicon.ico') EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' DEFAULT_FROM_EMAIL = 'registration@edx.org' DEFAULT_FEEDBACK_EMAIL = 'feedback@edx.org' +SERVER_EMAIL = 'devops@edx.org' ADMINS = ( ('edX Admins', 'admin@edx.org'), ) @@ -352,6 +354,14 @@ WIKI_LINK_DEFAULT_LEVEL = 2 PEARSONVUE_SIGNINPAGE_URL = "https://www1.pearsonvue.com/testtaker/signin/SignInPage/EDX" # TESTCENTER_ACCOMMODATION_REQUEST_EMAIL = "exam-help@edx.org" +##### Feedback submission mechanism ##### +FEEDBACK_SUBMISSION_EMAIL = None + +##### Zendesk ##### +ZENDESK_URL = None +ZENDESK_USER = None +ZENDESK_API_KEY = None + ################################# open ended grading config ##################### #By setting up the default settings with an incorrect user name and password, @@ -613,3 +623,4 @@ INSTALLED_APPS = ( # Discussion forums 'django_comment_client', ) + diff --git a/lms/envs/jasmine.py b/lms/envs/jasmine.py index f3f20e7fbc..ba4fcc5261 100644 --- a/lms/envs/jasmine.py +++ b/lms/envs/jasmine.py @@ -33,6 +33,6 @@ PIPELINE_JS['spec'] = { JASMINE_TEST_DIRECTORY = PROJECT_ROOT + '/static/coffee' -STATICFILES_DIRS.append(COMMON_ROOT / 'test' / 'phantom-jasmine' / 'lib') +STATICFILES_DIRS.append(REPO_ROOT/'node_modules/phantom-jasmine/lib') INSTALLED_APPS += ('django_jasmine', ) diff --git a/lms/lib/comment_client/utils.py b/lms/lib/comment_client/utils.py index 860035dc06..1ce03ed3c7 100644 --- a/lms/lib/comment_client/utils.py +++ b/lms/lib/comment_client/utils.py @@ -1,3 +1,4 @@ +from dogapi import dog_stats_api import json import logging import requests @@ -30,13 +31,23 @@ def merge_dict(dic1, dic2): def perform_request(method, url, data_or_params=None, *args, **kwargs): if data_or_params is None: data_or_params = {} + tags = [ + "{k}:{v}".format(k=k, v=v) + for (k, v) in data_or_params.items() + [("method", method), ("url", url)] + if k != 'api_key' + ] data_or_params['api_key'] = settings.API_KEY try: - if method in ['post', 'put', 'patch']: - response = requests.request(method, url, data=data_or_params, timeout=5) - else: - response = requests.request(method, url, params=data_or_params, timeout=5) + with dog_stats_api.timer('comment_client.request.time', tags=tags): + if method in ['post', 'put', 'patch']: + response = requests.request(method, url, data=data_or_params, timeout=5) + else: + response = requests.request(method, url, params=data_or_params, timeout=5) except Exception as err: + # remove API key if it is in the params + if 'api_key' in data_or_params: + log.info('Deleting API key from params') + del data_or_params['api_key'] log.exception("Trying to call {method} on {url} with params {params}".format( method=method, url=url, params=data_or_params)) # Reraise with a single exception type diff --git a/lms/static/images/university/epfl/epfl-cover.jpg b/lms/static/images/university/epfl/epfl-cover.jpg index 42b188c925..e585966e7b 100644 Binary files a/lms/static/images/university/epfl/epfl-cover.jpg and b/lms/static/images/university/epfl/epfl-cover.jpg differ diff --git a/lms/static/sass/base/_base.scss b/lms/static/sass/base/_base.scss index ca56f542d6..d2d4a0564f 100644 --- a/lms/static/sass/base/_base.scss +++ b/lms/static/sass/base/_base.scss @@ -202,5 +202,62 @@ mark { } } +.help-tab { + @include transform(rotate(-90deg)); + @include transform-origin(0 0); + top: 50%; + left: 0; + position: fixed; + z-index: 99; + a:link, a:visited { + cursor: pointer; + border: 1px solid #ccc; + border-top-style: none; + @include border-radius(0px 0px 10px 10px); + background: transparentize(#fff, 0.25); + color: transparentize(#333, 0.25); + font-weight: bold; + text-decoration: none; + padding: 6px 22px 11px; + display: inline-block; + &:hover { + color: #fff; + background: #1D9DD9; + } + } +} + +.help-buttons { + padding: 10px 50px; + + a:link, a:visited { + padding: 15px 0px; + text-align: center; + cursor: pointer; + background: #fff; + text-decoration: none; + display: block; + border: 1px solid #ccc; + + &#feedback_link_problem { + border-bottom-style: none; + @include border-radius(10px 10px 0px 0px); + } + + &#feedback_link_question { + border-top-style: none; + @include border-radius(0px 0px 10px 10px); + } + + &:hover { + color: #fff; + background: #1D9DD9; + } + } +} + +#feedback_form textarea[name="details"] { + height: 150px; +} diff --git a/lms/static/sass/shared/_modal.scss b/lms/static/sass/shared/_modal.scss index bfa803fee2..8ff58c1c14 100644 --- a/lms/static/sass/shared/_modal.scss +++ b/lms/static/sass/shared/_modal.scss @@ -149,13 +149,13 @@ } label { - color: #999; + color: #646464; &.field-error { display: block; color: #8F0E0E; - + input { + + input, + textarea { border: 1px solid #CA1111; color: #8F0E0E; } diff --git a/lms/templates/courseware/courseware.html b/lms/templates/courseware/courseware.html index 33dc9562a7..4889da25ca 100644 --- a/lms/templates/courseware/courseware.html +++ b/lms/templates/courseware/courseware.html @@ -156,7 +156,7 @@
- +
Hints @@ -176,8 +176,8 @@
- - + +
diff --git a/lms/templates/forgot_password_modal.html b/lms/templates/forgot_password_modal.html index 97b41d4be9..f0f571a817 100644 --- a/lms/templates/forgot_password_modal.html +++ b/lms/templates/forgot_password_modal.html @@ -12,19 +12,19 @@
- - + +
-
+

✕

-
+ @@ -40,5 +40,10 @@ $('#pwd_error').stop().css("display", "block"); } }); + + // removing close link's default behavior + $('#login-modal .close-modal').click(function(e) { + e.preventDefault(); + }); })(this) diff --git a/lms/templates/help_modal.html b/lms/templates/help_modal.html new file mode 100644 index 0000000000..deebd391d2 --- /dev/null +++ b/lms/templates/help_modal.html @@ -0,0 +1,176 @@ +<%namespace name='static' file='static_content.html'/> +<%! from datetime import datetime %> +<%! import pytz %> +<%! from django.conf import settings %> +<%! from courseware.tabs import get_discussion_link %> + +% if settings.MITX_FEATURES.get('ENABLE_FEEDBACK_SUBMISSION', False): + +
+ Help +
+ + + + + +%endif diff --git a/lms/templates/login_modal.html b/lms/templates/login_modal.html index 1587cca767..de1c437caf 100644 --- a/lms/templates/login_modal.html +++ b/lms/templates/login_modal.html @@ -9,14 +9,17 @@