Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #11938 from edx/andya/fix-underscore-on-sandboxes

Browse Source 
Fix issues with Underscore in the asset pipeline
This commit is contained in:
Andy Armstrong
2016-03-25 11:16:22 -04:00
parent 0ad9eb910e 6dd09a89cf
commit e8f620a2f4
20 changed files with 222 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -73,6 +73,7 @@ bin/
lms/static/css/
lms/static/certificates/css/
cms/static/css/
common/static/common/js/vendor/
### Styling generated from templates
lms/static/sass/*.css

2
cms/static/cms/js/require-config.js
View File

@@ -50,7 +50,7 @@
"moment": "js/vendor/moment.min",
"moment-with-locales": "js/vendor/moment-with-locales.min",
"text": 'js/vendor/requirejs/text',
"underscore": "js/vendor/underscore-min",
"underscore": "common/js/vendor/underscore",
"underscore.string": "js/vendor/underscore.string.min",
"backbone": "js/vendor/backbone-min",
"backbone-relational" : "js/vendor/backbone-relational.min",

2
cms/static/coffee/spec/main.coffee
View File

@@ -26,7 +26,7 @@ requirejs.config({
"moment": "xmodule_js/common_static/js/vendor/moment.min",
"moment-with-locales": "xmodule_js/common_static/js/vendor/moment-with-locales.min",
"text": "xmodule_js/common_static/js/vendor/requirejs/text",
"underscore": "xmodule_js/common_static/js/vendor/underscore-min",
"underscore": "xmodule_js/common_static/common/js/vendor/underscore",
"underscore.string": "xmodule_js/common_static/js/vendor/underscore.string.min",
"backbone": "xmodule_js/common_static/js/vendor/backbone-min",
"backbone.associations": "xmodule_js/common_static/js/vendor/backbone-associations-min",

2
cms/static/coffee/spec/main_squire.coffee
View File

@@ -22,7 +22,7 @@ requirejs.config({
"datepair": "xmodule_js/common_static/js/vendor/timepicker/datepair",
"date": "xmodule_js/common_static/js/vendor/date",
"text": "xmodule_js/common_static/js/vendor/requirejs/text",
"underscore": "xmodule_js/common_static/js/vendor/underscore-min",
"underscore": "xmodule_js/common_static/common/js/vendor/underscore",
"underscore.string": "xmodule_js/common_static/js/vendor/underscore.string.min",
"backbone": "xmodule_js/common_static/js/vendor/backbone-min",
"backbone.associations": "xmodule_js/common_static/js/vendor/backbone-associations-min",

2
cms/static/js_test.yml
View File

@@ -35,7 +35,7 @@ lib_paths:
- xmodule_js/common_static/js/vendor/jquery-ui.min.js
- xmodule_js/common_static/js/vendor/jquery.cookie.js
- xmodule_js/common_static/js/vendor/jquery.simulate.js
- xmodule_js/common_static/js/vendor/underscore-min.js
- xmodule_js/common_static/common/js/vendor/underscore.js
- xmodule_js/common_static/js/vendor/underscore.string.min.js
- xmodule_js/common_static/js/vendor/backbone-min.js
- xmodule_js/common_static/js/vendor/backbone-associations-min.js

2
cms/static/js_test_squire.yml
View File

@@ -34,7 +34,7 @@ lib_paths:
- xmodule_js/common_static/js/vendor/jquery.min.js
- xmodule_js/common_static/js/vendor/jquery-ui.min.js
- xmodule_js/common_static/js/vendor/jquery.cookie.js
- xmodule_js/common_static/js/vendor/underscore-min.js
- xmodule_js/common_static/common/js/vendor/underscore.js
- xmodule_js/common_static/js/vendor/underscore.string.min.js
- xmodule_js/common_static/js/vendor/backbone-min.js
- xmodule_js/common_static/js/vendor/backbone-associations-min.js

2
common/lib/xmodule/xmodule/js/js_test.yml
View File

@@ -45,7 +45,7 @@ lib_paths:
- common_static/js/vendor/jquery.ui.draggable.js
- common_static/js/vendor/jquery.cookie.js
- common_static/js/vendor/json2.js
- common_static/js/vendor/underscore-min.js
- common_static/common/js/vendor/underscore.js
- common_static/js/vendor/backbone-min.js
- common_static/js/vendor/jquery.leanModal.js
- common_static/js/vendor/CodeMirror/codemirror.js

2
common/static/common/js/spec/main_requirejs.js
View File

@@ -22,7 +22,7 @@
'jquery.url': 'js/vendor/url.min',
'sinon': 'js/vendor/sinon-1.17.0',
'text': 'js/vendor/requirejs/text',
'underscore': 'js/vendor/underscore-min',
'underscore': 'common/js/vendor/underscore',
'underscore.string': 'js/vendor/underscore.string.min',
'backbone': 'js/vendor/backbone-min',
'backbone.associations': 'js/vendor/backbone-associations-min',

1
common/static/js/vendor/underscore-min.js vendored
View File

@@ -1 +0,0 @@
../../../../node_modules/underscore/underscore-min.js

2
common/static/js_test.yml
View File

@@ -33,7 +33,7 @@ lib_paths:
- js/vendor/jasmine-imagediff.js
- js/vendor/jquery.truncate.js
- js/vendor/mustache.js
- js/vendor/underscore-min.js
- common/js/vendor/underscore.js
- js/vendor/underscore.string.min.js
- js/vendor/backbone-min.js
- js/vendor/jquery.timeago.js

2
lms/envs/common.py
View File

@@ -1242,7 +1242,7 @@ base_vendor_js = [
'js/vendor/jquery.min.js',
'js/vendor/jquery.cookie.js',
'js/vendor/url.min.js',
'js/vendor/underscore-min.js',
'common/js/vendor/underscore.js',
'js/vendor/underscore.string.min.js',
'js/vendor/requirejs/require.js',
'js/RequireJS-namespace-undefine.js',

2
lms/static/js/spec/main.js
View File

@@ -29,7 +29,7 @@
'moment': 'xmodule_js/common_static/js/vendor/moment.min',
'moment-with-locales': 'xmodule_js/common_static/js/vendor/moment-with-locales.min',
'text': 'xmodule_js/common_static/js/vendor/requirejs/text',
'underscore': 'xmodule_js/common_static/js/vendor/underscore-min',
'underscore': 'xmodule_js/common_static/common/js/vendor/underscore',
'underscore.string': 'xmodule_js/common_static/js/vendor/underscore.string.min',
'backbone': 'xmodule_js/common_static/js/vendor/backbone-min',
'backbone.associations': 'xmodule_js/common_static/js/vendor/backbone-associations-min',

2
lms/static/js_test.yml
View File

@@ -56,7 +56,7 @@ lib_paths:
- xmodule_js/src/video/
- xmodule_js/src/xmodule.js
- xmodule_js/common_static/js/src/
- xmodule_js/common_static/js/vendor/underscore-min.js
- xmodule_js/common_static/common/js/vendor/underscore.js
- xmodule_js/common_static/js/vendor/underscore.string.min.js
- xmodule_js/common_static/js/vendor/backbone-min.js
- xmodule_js/common_static/js/vendor/backbone.paginator.min.js

2
lms/static/lms/js/require-config.js
View File

@@ -47,7 +47,7 @@
"backbone": "js/vendor/backbone-min",
"backbone-super": "js/vendor/backbone-super",
"backbone.paginator": "js/vendor/backbone.paginator.min",
"underscore": "js/vendor/underscore-min",
"underscore": "common/js/vendor/underscore",
"underscore.string": "js/vendor/underscore.string.min",
"jquery": "js/vendor/jquery.min",
"jquery.cookie": "js/vendor/jquery.cookie",

30
pavelib/assets.py
View File

@@ -35,6 +35,15 @@ CMS_SASS_DIRECTORIES = [
THEME_SASS_DIRECTORIES = []
SASS_LOAD_PATHS = ['common/static', 'common/static/sass']
# A list of NPM installed libraries that should be copied into the common
# static directory.
NPM_INSTALLED_LIBRARIES = [
'underscore/underscore.js'
]
# Directory to install static vendor files
NPM_VENDOR_DIRECTORY = path("common/static/common/js/vendor")
def configure_paths():
"""Configure our paths based on settings. Called immediately."""
@@ -292,6 +301,26 @@ def compile_templated_sass(systems, settings):
print("\t\tFinished preprocessing {} assets.".format(system))
def process_npm_assets():
"""
Process vendor libraries installed via NPM.
"""
# Skip processing of the libraries if this is just a dry run
if tasks.environment.dry_run:
tasks.environment.info("install npm_assets")
return
# Ensure that the vendor directory exists
NPM_VENDOR_DIRECTORY.mkdir_p()
# Copy each file to the vendor directory, overwriting any existing file.
for library in NPM_INSTALLED_LIBRARIES:
sh('/bin/cp -rf node_modules/{library} {vendor_dir}'.format(
library=library,
vendor_dir=NPM_VENDOR_DIRECTORY,
))
def process_xmodule_assets():
"""
Process XModule static assets.
@@ -387,6 +416,7 @@ def update_assets(args):
compile_templated_sass(args.system, args.settings)
process_xmodule_assets()
process_npm_assets()
compile_coffeescript()
call_task('pavelib.assets.compile_sass', options={'system': args.system, 'debug': args.debug})

134
pavelib/paver_tests/test_js_test.py Normal file
View File

@@ -0,0 +1,134 @@
"""Unit tests for the Paver JavaScript testing tasks."""
import ddt
from mock import patch
from paver.easy import call_task
import pavelib.js_test
from .utils import PaverTestCase
@ddt.ddt
class TestPaverJavaScriptTestTasks(PaverTestCase):
"""
Test the Paver JavaScript testing tasks.
"""
EXPECTED_DELETE_JAVASCRIPT_REPORT_COMMAND = u'find {platform_root}/reports/javascript -type f -delete'
EXPECTED_INSTALL_NPM_ASSETS_COMMAND = u'install npm_assets'
EXPECTED_COFFEE_COMMAND = (
u'node_modules/.bin/coffee --compile `find {platform_root}/lms {platform_root}/cms '
u'{platform_root}/common -type f -name "*.coffee"`'
)
EXPECTED_JS_TEST_TOOL_OPTIONS = (
u"{platform_root}/lms/static/js_test.yml "
u"{platform_root}/lms/static/js_test_coffee.yml "
u"{platform_root}/cms/static/js_test.yml "
u"{platform_root}/cms/static/js_test_squire.yml "
u"{platform_root}/common/lib/xmodule/xmodule/js/js_test.yml "
u"{platform_root}/common/static/js_test.yml "
u"{platform_root}/common/static/js_test_requirejs.yml "
u"--use-firefox "
u"--timeout-sec 600 "
u"--xunit-report "
u"{platform_root}/reports/javascript/javascript_xunit.xml"
)
EXPECTED_COVERAGE_OPTIONS = (
u' --coverage-xml {platform_root}/reports/javascript/coverage.xml'
)
EXPECTED_COMMANDS = [
u"make report_dir",
u'git clean -fqdx test_root/logs test_root/data test_root/staticfiles test_root/uploads',
u"find . -name '.git' -prune -o -name '*.pyc' -exec rm {} \\;",
u'rm -rf test_root/log/auto_screenshots/*',
u"rm -rf /tmp/mako_[cl]ms",
]
def setUp(self):
super(TestPaverJavaScriptTestTasks, self).setUp()
# Mock the paver @needs decorator
self._mock_paver_needs = patch.object(pavelib.js_test.test_js, 'needs').start()
self._mock_paver_needs.return_value = 0
# Cleanup mocks
self.addCleanup(self._mock_paver_needs.stop)
@ddt.data(
[""],
["--coverage"],
["--suite=lms"],
["--suite=lms --coverage"],
)
@ddt.unpack
def test_test_js_run(self, options_string):
"""
Test the "test_js_run" task.
"""
options = self.parse_options_string(options_string)
self.reset_task_messages()
call_task("pavelib.js_test.test_js_run", options=options)
self.verify_messages(options=options, dev_mode=False)
@ddt.data(
[""],
["--port=9999"],
["--suite=lms"],
["--suite=lms --port=9999"],
)
@ddt.unpack
def test_test_js_dev(self, options_string):
"""
Test the "test_js_run" task.
"""
options = self.parse_options_string(options_string)
self.reset_task_messages()
call_task("pavelib.js_test.test_js_dev", options=options)
self.verify_messages(options=options, dev_mode=True)
def parse_options_string(self, options_string):
"""
Parse a string containing the options for a test run
"""
parameters = options_string.split(" ")
suite = "all"
if "--system=lms" in parameters:
suite = "lms"
elif "--system=common" in parameters:
suite = "common"
coverage = "--coverage" in parameters
port = None
if "--port=9999" in parameters:
port = 9999
return {
"suite": suite,
"coverage": coverage,
"port": port,
}
def verify_messages(self, options, dev_mode):
"""
Verify that the messages generated when running tests are as expected
for the specified options and dev_mode.
"""
is_coverage = options['coverage']
port = options['port']
expected_messages = []
expected_messages.extend(self.EXPECTED_COMMANDS)
if not dev_mode and not is_coverage:
expected_messages.append(self.EXPECTED_DELETE_JAVASCRIPT_REPORT_COMMAND.format(
platform_root=self.platform_root
))
expected_messages.append(self.EXPECTED_INSTALL_NPM_ASSETS_COMMAND)
expected_messages.append(self.EXPECTED_COFFEE_COMMAND.format(platform_root=self.platform_root))
expected_test_tool_command = u'js-test-tool {command} {options}'.format(
command='dev' if dev_mode else 'run',
options=self.EXPECTED_JS_TEST_TOOL_OPTIONS.format(platform_root=self.platform_root),
)
if is_coverage:
expected_test_tool_command += self.EXPECTED_COVERAGE_OPTIONS.format(platform_root=self.platform_root)
if port:
expected_test_tool_command += u" -p {port}".format(port=port)
expected_messages.append(expected_test_tool_command)
self.assertEquals(self.task_messages, expected_messages)

41
pavelib/paver_tests/test_servers.py
View File

@@ -1,44 +1,43 @@
"""Unit tests for the Paver server tasks."""
import ddt
import os
from paver.easy import call_task
from .utils import PaverTestCase
EXPECTED_COFFEE_COMMAND = (
"node_modules/.bin/coffee --compile `find {platform_root}/lms "
"{platform_root}/cms {platform_root}/common -type f -name \"*.coffee\"`"
u"node_modules/.bin/coffee --compile `find {platform_root}/lms "
u"{platform_root}/cms {platform_root}/common -type f -name \"*.coffee\"`"
)
EXPECTED_SASS_COMMAND = (
"libsass {sass_directory}"
u"libsass {sass_directory}"
)
EXPECTED_COMMON_SASS_DIRECTORIES = [
"common/static/sass",
u"common/static/sass",
]
EXPECTED_LMS_SASS_DIRECTORIES = [
"lms/static/sass",
"lms/static/themed_sass",
"lms/static/certificates/sass",
u"lms/static/sass",
u"lms/static/themed_sass",
u"lms/static/certificates/sass",
]
EXPECTED_CMS_SASS_DIRECTORIES = [
"cms/static/sass",
u"cms/static/sass",
]
EXPECTED_PREPROCESS_ASSETS_COMMAND = (
"python manage.py {system} --settings={asset_settings} preprocess_assets"
" {system}/static/sass/*.scss {system}/static/themed_sass"
u"python manage.py {system} --settings={asset_settings} preprocess_assets"
u" {system}/static/sass/*.scss {system}/static/themed_sass"
)
EXPECTED_COLLECT_STATIC_COMMAND = (
"python manage.py {system} --settings={asset_settings} collectstatic --noinput > /dev/null"
u"python manage.py {system} --settings={asset_settings} collectstatic --noinput > /dev/null"
)
EXPECTED_CELERY_COMMAND = (
"python manage.py lms --settings={settings} celery worker --beat --loglevel=INFO --pythonpath=."
u"python manage.py lms --settings={settings} celery worker --beat --loglevel=INFO --pythonpath=."
)
EXPECTED_RUN_SERVER_COMMAND = (
"python manage.py {system} --settings={settings} runserver --traceback --pythonpath=. 0.0.0.0:{port}"
u"python manage.py {system} --settings={settings} runserver --traceback --pythonpath=. 0.0.0.0:{port}"
)
EXPECTED_INDEX_COURSE_COMMAND = (
"python manage.py {system} --settings={settings} reindex_course --setup"
u"python manage.py {system} --settings={settings} reindex_course --setup"
)
@@ -227,13 +226,13 @@ class TestPaverServerTasks(PaverTestCase):
expected_settings = "devstack_optimized"
expected_asset_settings = "test_static_optimized"
expected_collect_static = not is_fast and expected_settings != "devstack"
platform_root = os.getcwd()
if not is_fast:
expected_messages.append(EXPECTED_PREPROCESS_ASSETS_COMMAND.format(
system=system, asset_settings=expected_asset_settings
))
expected_messages.append("xmodule_assets common/static/xmodule")
expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=platform_root))
expected_messages.append(u"xmodule_assets common/static/xmodule")
expected_messages.append(u"install npm_assets")
expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=self.platform_root))
expected_messages.extend(self.expected_sass_commands(system=system))
if expected_collect_static:
expected_messages.append(EXPECTED_COLLECT_STATIC_COMMAND.format(
@@ -265,7 +264,6 @@ class TestPaverServerTasks(PaverTestCase):
expected_settings = "devstack_optimized"
expected_asset_settings = "test_static_optimized"
expected_collect_static = not is_fast and expected_settings != "devstack"
platform_root = os.getcwd()
expected_messages = []
if not is_fast:
expected_messages.append(EXPECTED_PREPROCESS_ASSETS_COMMAND.format(
@@ -274,8 +272,9 @@ class TestPaverServerTasks(PaverTestCase):
expected_messages.append(EXPECTED_PREPROCESS_ASSETS_COMMAND.format(
system="cms", asset_settings=expected_asset_settings
))
expected_messages.append("xmodule_assets common/static/xmodule")
expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=platform_root))
expected_messages.append(u"xmodule_assets common/static/xmodule")
expected_messages.append(u"install npm_assets")
expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=self.platform_root))
expected_messages.extend(self.expected_sass_commands())
if expected_collect_static:
expected_messages.append(EXPECTED_COLLECT_STATIC_COMMAND.format(

7
pavelib/paver_tests/utils.py
View File

@@ -31,6 +31,11 @@ class PaverTestCase(TestCase):
"""Returns the messages output by the Paver task."""
return tasks.environment.messages
@property
def platform_root(self):
"""Returns the current platform's root directory."""
return os.getcwd()
def reset_task_messages(self):
"""Clear the recorded message"""
tasks.environment.messages = []
@@ -52,4 +57,4 @@ class MockEnvironment(tasks.Environment):
else:
output = message
if not output.startswith("--->"):
self.messages.append(output)
self.messages.append(unicode(output))

9
pavelib/utils/test/suites/js_suite.py
View File

@@ -1,6 +1,9 @@
"""
Javascript test tasks
"""
from paver import tasks
from pavelib import assets
from pavelib.utils.test import utils as test_utils
from pavelib.utils.test.suites.suite import TestSuite
@@ -31,13 +34,17 @@ class JsTestSuite(TestSuite):
def __enter__(self):
super(JsTestSuite, self).__enter__()
self.report_dir.makedirs_p()
if tasks.environment.dry_run:
tasks.environment.info("make report_dir")
else:
self.report_dir.makedirs_p()
if not self.skip_clean:
test_utils.clean_test_files()
if self.mode == 'run' and not self.run_under_coverage:
test_utils.clean_dir(self.report_dir)
assets.process_npm_assets()
assets.compile_coffeescript("`find lms cms common -type f -name \"*.coffee\"`")
@property

11
pavelib/utils/test/suites/suite.py
View File

@@ -3,6 +3,8 @@ A class used for defining and running test suites
"""
import sys
import subprocess
from paver import tasks
from paver.easy import sh
from pavelib.utils.process import kill_process
@@ -74,6 +76,11 @@ class TestSuite(object):
returns True.
"""
cmd = self.cmd
if tasks.environment.dry_run:
tasks.environment.info(cmd)
return
sys.stdout.write(cmd)
msg = colorize(
@@ -130,6 +137,10 @@ class TestSuite(object):
Runs the tests in the suite while tracking and reporting failures.
"""
self.run_suite_tests()
if tasks.environment.dry_run:
return
self.report_test_results()
if len(self.failed_suites) > 0: