From 9c7ba41880827d05e3bde0cf8a5ea8a5960a0be6 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Fri, 12 Apr 2013 12:06:38 -0400 Subject: [PATCH 01/15] Only add the xmodule.coffee file once per class, and put it before all other coffeescript files --- common/lib/xmodule/xmodule/x_module.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index e6d367ac7a..ab3fe8c027 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -45,17 +45,13 @@ class HTMLSnippet(object): # cdodge: We've moved the xmodule.coffee script from an outside directory into the xmodule area of common # this means we need to make sure that all xmodules include this dependency which had been previously implicitly # fulfilled in a different area of code - js = cls.js + coffee = cls.js.setdefault('coffee', []) + fragment = resource_string(__name__, 'js/src/xmodule.coffee') - if js is None: - js = {} + if fragment not in coffee: + coffee.insert(0, fragment) - if 'coffee' not in js: - js['coffee'] = [] - - js['coffee'].append(resource_string(__name__, 'js/src/xmodule.coffee')) - - return js + return cls.js @classmethod def get_css(cls): From 346aee8863514f89f278c1014a0a818c3f9257df Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Fri, 12 Apr 2013 12:23:13 -0400 Subject: [PATCH 02/15] Make django-pipeline only work with raw js and css --- cms/envs/common.py | 33 ++++++++----------------- cms/envs/jasmine.py | 2 +- common/lib/xmodule/xmodule/x_module.py | 2 +- lms/envs/common.py | 34 ++++++++++---------------- 4 files changed, 25 insertions(+), 46 deletions(-) diff --git a/cms/envs/common.py b/cms/envs/common.py index ca08a56a70..21995be40d 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -35,7 +35,7 @@ MITX_FEATURES = { 'AUTH_USE_MIT_CERTIFICATES': False, 'STUB_VIDEO_FOR_TESTING': False, # do not display video when running automated acceptance tests 'STAFF_EMAIL': '', # email address for staff (eg to request course creation) - 'STUDIO_NPS_SURVEY': True, + 'STUDIO_NPS_SURVEY': True, 'SEGMENT_IO': True, } ENABLE_JASMINE = False @@ -195,20 +195,21 @@ from rooted_paths import rooted_glob, remove_root write_descriptor_styles(PROJECT_ROOT / "static/sass/descriptor", [RawDescriptor, ErrorDescriptor]) write_module_styles(PROJECT_ROOT / "static/sass/module", [RawDescriptor, ErrorDescriptor]) -descriptor_js = remove_root( +descriptor_js = [path.replace('.coffee', '.js') for path in remove_root( PROJECT_ROOT / 'static', write_descriptor_js( PROJECT_ROOT / "static/coffee/descriptor", [RawDescriptor, ErrorDescriptor] ) -) -module_js = remove_root( +)] + +module_js = [path.replace('.coffee', '.js') for path in remove_root( PROJECT_ROOT / 'static', write_module_js( PROJECT_ROOT / "static/coffee/module", [RawDescriptor, ErrorDescriptor] ) -) +)] PIPELINE_CSS = { 'base-style': { @@ -216,19 +217,17 @@ PIPELINE_CSS = { 'js/vendor/CodeMirror/codemirror.css', 'css/vendor/ui-lightness/jquery-ui-1.8.22.custom.css', 'css/vendor/jquery.qtip.min.css', - 'sass/base-style.scss' + 'sass/base-style.css' ], 'output_filename': 'css/cms-base-style.css', }, } -PIPELINE_ALWAYS_RECOMPILE = ['sass/base-style.scss'] - PIPELINE_JS = { 'main': { 'source_filenames': sorted( - rooted_glob(COMMON_ROOT / 'static/', 'coffee/src/**/*.coffee') + - rooted_glob(PROJECT_ROOT / 'static/', 'coffee/src/**/*.coffee') + rooted_glob(COMMON_ROOT / 'static/', 'coffee/src/**/*.js') + + rooted_glob(PROJECT_ROOT / 'static/', 'coffee/src/**/*.js') ) + ['js/hesitate.js', 'js/base.js'], 'output_filename': 'js/cms-application.js', }, @@ -237,18 +236,11 @@ PIPELINE_JS = { 'output_filename': 'js/cms-modules.js', }, 'spec': { - 'source_filenames': sorted(rooted_glob(PROJECT_ROOT / 'static/', 'coffee/spec/**/*.coffee')), + 'source_filenames': sorted(rooted_glob(PROJECT_ROOT / 'static/', 'coffee/spec/**/*.js')), 'output_filename': 'js/cms-spec.js' } } -PIPELINE_COMPILERS = [ - 'pipeline.compilers.sass.SASSCompiler', - 'pipeline.compilers.coffee.CoffeeScriptCompiler', -] - -PIPELINE_SASS_ARGUMENTS = '-t compressed -r {proj_dir}/static/sass/bourbon/lib/bourbon.rb'.format(proj_dir=PROJECT_ROOT) - PIPELINE_CSS_COMPRESSOR = None PIPELINE_JS_COMPRESSOR = None @@ -260,11 +252,6 @@ STATICFILES_IGNORE_PATTERNS = ( ) PIPELINE_YUI_BINARY = 'yui-compressor' -PIPELINE_SASS_BINARY = 'sass' -PIPELINE_COFFEE_SCRIPT_BINARY = 'coffee' - -# Setting that will only affect the MITx version of django-pipeline until our changes are merged upstream -PIPELINE_COMPILE_INPLACE = True ############################ APPS ##################################### diff --git a/cms/envs/jasmine.py b/cms/envs/jasmine.py index 5c9be1cf9c..ac28f8fc9a 100644 --- a/cms/envs/jasmine.py +++ b/cms/envs/jasmine.py @@ -27,7 +27,7 @@ PIPELINE_JS['js-test-source'] = { } PIPELINE_JS['spec'] = { - 'source_filenames': sorted(rooted_glob(PROJECT_ROOT / 'static/', 'coffee/spec/**/*.coffee')), + 'source_filenames': sorted(rooted_glob(PROJECT_ROOT / 'static/', 'coffee/spec/**/*.js')), 'output_filename': 'js/cms-spec.js' } diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index ab3fe8c027..1fd0b8e138 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -337,7 +337,7 @@ class XModuleDescriptor(XModuleFields, HTMLSnippet, ResourceTemplates, XBlock): # cdodge: this is a list of metadata names which are 'system' metadata # and should not be edited by an end-user - system_metadata_fields = ['data_dir', 'published_date', 'published_by', 'is_draft', + system_metadata_fields = ['data_dir', 'published_date', 'published_by', 'is_draft', 'discussion_id', 'xml_attributes'] # A list of descriptor attributes that must be equal for the descriptors to diff --git a/lms/envs/common.py b/lms/envs/common.py index 8654b5ebf5..27c2cfc022 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -394,17 +394,18 @@ from xmodule.hidden_module import HiddenDescriptor from rooted_paths import rooted_glob, remove_root write_module_styles(PROJECT_ROOT / 'static/sass/module', [HiddenDescriptor]) -module_js = remove_root( + +module_js = [path.replace('.coffee', '.js') for path in remove_root( PROJECT_ROOT / 'static', write_module_js(PROJECT_ROOT / 'static/coffee/module', [HiddenDescriptor]) -) +)] courseware_js = ( [ - 'coffee/src/' + pth + '.coffee' + 'coffee/src/' + pth + '.js' for pth in ['courseware', 'histogram', 'navigation', 'time'] ] + - sorted(rooted_glob(PROJECT_ROOT / 'static', 'coffee/src/modules/**/*.coffee')) + sorted(rooted_glob(PROJECT_ROOT / 'static', 'coffee/src/modules/**/*.js')) ) # 'js/vendor/RequireJS.js' - Require JS wrapper. @@ -420,13 +421,13 @@ main_vendor_js = [ 'js/vendor/jquery.ba-bbq.min.js', ] -discussion_js = sorted(rooted_glob(PROJECT_ROOT / 'static', 'coffee/src/discussion/**/*.coffee')) -staff_grading_js = sorted(rooted_glob(PROJECT_ROOT / 'static', 'coffee/src/staff_grading/**/*.coffee')) -open_ended_js = sorted(rooted_glob(PROJECT_ROOT / 'static', 'coffee/src/open_ended/**/*.coffee')) +discussion_js = sorted(rooted_glob(PROJECT_ROOT / 'static', 'coffee/src/discussion/**/*.js')) +staff_grading_js = sorted(rooted_glob(PROJECT_ROOT / 'static', 'coffee/src/staff_grading/**/*.js')) +open_ended_js = sorted(rooted_glob(PROJECT_ROOT / 'static', 'coffee/src/open_ended/**/*.js')) PIPELINE_CSS = { 'application': { - 'source_filenames': ['sass/application.scss'], + 'source_filenames': ['sass/application.css'], 'output_filename': 'css/lms-application.css', }, 'course': { @@ -435,24 +436,23 @@ PIPELINE_CSS = { 'css/vendor/jquery.treeview.css', 'css/vendor/ui-lightness/jquery-ui-1.8.22.custom.css', 'css/vendor/jquery.qtip.min.css', - 'sass/course.scss' + 'sass/course.css' ], 'output_filename': 'css/lms-course.css', }, 'ie-fixes': { - 'source_filenames': ['sass/ie.scss'], + 'source_filenames': ['sass/ie.css'], 'output_filename': 'css/lms-ie.css', }, } -PIPELINE_ALWAYS_RECOMPILE = ['sass/application.scss', 'sass/ie.scss', 'sass/course.scss'] PIPELINE_JS = { 'application': { # Application will contain all paths not in courseware_only_js 'source_filenames': sorted( - set(rooted_glob(COMMON_ROOT / 'static', 'coffee/src/**/*.coffee') + - rooted_glob(PROJECT_ROOT / 'static', 'coffee/src/**/*.coffee')) - + set(rooted_glob(COMMON_ROOT / 'static', 'coffee/src/**/*.js') + + rooted_glob(PROJECT_ROOT / 'static', 'coffee/src/**/*.js')) - set(courseware_js + discussion_js + staff_grading_js + open_ended_js) ) + [ 'js/form.ext.js', @@ -510,12 +510,6 @@ if os.path.isdir(DATA_DIR): os.system("rm %s" % (js_dir / new_filename)) os.system("coffee -c %s" % (js_dir / filename)) -PIPELINE_COMPILERS = [ - 'pipeline.compilers.sass.SASSCompiler', - 'pipeline.compilers.coffee.CoffeeScriptCompiler', -] - -PIPELINE_SASS_ARGUMENTS = '-t compressed -r {proj_dir}/static/sass/bourbon/lib/bourbon.rb'.format(proj_dir=PROJECT_ROOT) PIPELINE_CSS_COMPRESSOR = None PIPELINE_JS_COMPRESSOR = None @@ -526,8 +520,6 @@ STATICFILES_IGNORE_PATTERNS = ( ) PIPELINE_YUI_BINARY = 'yui-compressor' -PIPELINE_SASS_BINARY = 'sass' -PIPELINE_COFFEE_SCRIPT_BINARY = 'coffee' # Setting that will only affect the MITx version of django-pipeline until our changes are merged upstream PIPELINE_COMPILE_INPLACE = True From 291e772ce06bc640f0b0936c8c6aede6218a59a4 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Fri, 12 Apr 2013 12:23:31 -0400 Subject: [PATCH 03/15] Clean up cruft from rakefile --- rakefile | 64 -------------------------------------------------------- 1 file changed, 64 deletions(-) diff --git a/rakefile b/rakefile index d6571118ca..e56cbed332 100644 --- a/rakefile +++ b/rakefile @@ -21,15 +21,6 @@ COMMIT = (ENV["GIT_COMMIT"] || `git rev-parse HEAD`).chomp()[0, 10] BRANCH = (ENV["GIT_BRANCH"] || `git symbolic-ref -q HEAD`).chomp().gsub('refs/heads/', '').gsub('origin/', '') BUILD_NUMBER = (ENV["BUILD_NUMBER"] || "dev").chomp() -if BRANCH == "master" - DEPLOY_NAME = "#{PACKAGE_NAME}-#{BUILD_NUMBER}-#{COMMIT}" -else - DEPLOY_NAME = "#{PACKAGE_NAME}-#{BRANCH}-#{BUILD_NUMBER}-#{COMMIT}" -end -PACKAGE_REPO = "packages@gp.mitx.mit.edu:/opt/pkgrepo.incoming" - -NORMALIZED_DEPLOY_NAME = DEPLOY_NAME.downcase().gsub(/[_\/]/, '-') -INSTALL_DIR_PATH = File.join(DEPLOY_DIR, NORMALIZED_DEPLOY_NAME) # Set up the clean and clobber tasks CLOBBER.include(BUILD_DIR, REPORT_DIR, 'test_root/*_repo', 'test_root/staticfiles') CLEAN.include("#{BUILD_DIR}/*.deb", "#{BUILD_DIR}/util") @@ -360,51 +351,6 @@ task :set_staff, [:user, :system, :env] do |t, args| sh(django_admin(args.system, args.env, 'set_staff', args.user)) end -task :package do - FileUtils.mkdir_p(BUILD_DIR) - - Dir.chdir(BUILD_DIR) do - afterremove = Tempfile.new('afterremove') - afterremove.write <<-AFTERREMOVE.gsub(/^\s*/, '') - #! /bin/bash - set -e - set -x - - # to be a little safer this rm is executed - # as the makeitso user - - if [[ -d "#{INSTALL_DIR_PATH}" ]]; then - sudo rm -rf "#{INSTALL_DIR_PATH}" - fi - - AFTERREMOVE - afterremove.close() - FileUtils.chmod(0755, afterremove.path) - - args = ["fakeroot", "fpm", "-s", "dir", "-t", "deb", - "--after-remove=#{afterremove.path}", - "--prefix=#{INSTALL_DIR_PATH}", - "--exclude=**/build/**", - "--exclude=**/rakefile", - "--exclude=**/.git/**", - "--exclude=**/*.pyc", - "--exclude=**/reports/**", - "--exclude=**/test_root/**", - "--exclude=**/.coverage/**", - "-C", "#{REPO_ROOT}", - "--provides=#{PACKAGE_NAME}", - "--name=#{NORMALIZED_DEPLOY_NAME}", - "--version=#{PKG_VERSION}", - "-a", "all", - "."] - system(*args) || raise("fpm failed to build the .deb") - end -end - -task :publish => :package do - sh("scp #{BUILD_DIR}/#{NORMALIZED_DEPLOY_NAME}_#{PKG_VERSION}*.deb #{PACKAGE_REPO}") -end - namespace :cms do desc "Clone existing MongoDB based course" task :clone do @@ -415,9 +361,7 @@ namespace :cms do raise "You must pass in a SOURCE_LOC and DEST_LOC parameters" end end -end -namespace :cms do desc "Delete existing MongoDB based course" task :delete_course do @@ -429,9 +373,7 @@ namespace :cms do raise "You must pass in a LOC parameter" end end -end -namespace :cms do desc "Import course data within the given DATA_DIR variable" task :import do if ENV['DATA_DIR'] and ENV['COURSE_DIR'] @@ -443,16 +385,12 @@ namespace :cms do "Example: \`rake cms:import DATA_DIR=../data\`" end end -end -namespace :cms do desc "Imports all the templates from the code pack" task :update_templates do sh(django_admin(:cms, :dev, :update_templates)) end -end -namespace :cms do desc "Import course data within the given DATA_DIR variable" task :xlint do if ENV['DATA_DIR'] and ENV['COURSE_DIR'] @@ -464,9 +402,7 @@ namespace :cms do "Example: \`rake cms:import DATA_DIR=../data\`" end end -end -namespace :cms do desc "Export course data to a tar.gz file" task :export do if ENV['COURSE_ID'] and ENV['OUTPUT_PATH'] From 589267599f6b7c613b0fd1ad4fd94c30fabb54b9 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Fri, 12 Apr 2013 15:29:53 -0400 Subject: [PATCH 04/15] Run coffee and sass in watch mode when running lms or cms, and oneshot before running collectstatic --- rakefile | 70 +++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 16 deletions(-) diff --git a/rakefile b/rakefile index e56cbed332..7d26c027ae 100644 --- a/rakefile +++ b/rakefile @@ -34,16 +34,42 @@ def django_admin(system, env, command, *args) return "#{django_admin} #{command} --traceback --settings=#{system}.envs.#{env} --pythonpath=. #{args.join(' ')}" end +# Runs Process.spawn, and kills the process at the end of the rake process +# Expects the same arguments as Process.spawn +def background_process(*command) + pid = Process.spawn(*command, :pgroup => true) + + at_exit do + puts "Ending process and children" + pgid = Process.getpgid(pid) + begin + Timeout.timeout(5) do + puts "Terminating process group #{pgid}" + Process.kill(:SIGTERM, -pgid) + puts "Waiting on process group #{pgid}" + Process.wait(-pgid) + puts "Done waiting on process group #{pgid}" + end + rescue Timeout::Error + puts "Killing process group #{pgid}" + Process.kill(:SIGKILL, -pgid) + puts "Waiting on process group #{pgid}" + Process.wait(-pgid) + puts "Done waiting on process group #{pgid}" + end + end +end + def django_for_jasmine(system, django_reload) if !django_reload reload_arg = '--noreload' end port = 10000 + rand(40000) - django_pid = fork do - exec(*django_admin(system, 'jasmine', 'runserver', '-v', '0', port.to_s, reload_arg).split(' ')) - end jasmine_url = "http://localhost:#{port}/_jasmine/" + + background_process(*django_admin(system, 'jasmine', 'runserver', '-v', '0', port.to_s, reload_arg).split(' ')) + up = false start_time = Time.now until up do @@ -61,16 +87,7 @@ def django_for_jasmine(system, django_reload) sleep(0.5) end end - begin - yield jasmine_url - ensure - if django_reload - Process.kill(:SIGKILL, -Process.getpgid(django_pid)) - else - Process.kill(:SIGKILL, django_pid) - end - Process.wait(django_pid) - end + yield jasmine_url end def template_jasmine_runner(lib) @@ -102,6 +119,25 @@ def report_dir_path(dir) return File.join(REPORT_DIR, dir.to_s) end +def compile_assets(watch=false) + coffee_cmd = "coffee #{watch ? '--watch' : ''} --compile */static" + sass_cmd = "sass --style compressed --require ./common/static/sass/bourbon/lib/bourbon.rb #{watch ? '--watch' : '--update'} */static" + + if watch + background_process(coffee_cmd) + background_process(sass_cmd) + else + coffee_pid = Process.spawn(coffee_cmd) + puts "Waiting for coffee to complete (pid #{coffee_pid})" + Process.wait(coffee_pid) + puts "Coffee completed" + sass_pid = Process.spawn(sass_cmd) + puts "Waiting for sass to complete (pid #{sass_pid})" + Process.wait(sass_pid) + puts "Sass completed" + end +end + task :default => [:test, :pep8, :pylint] directory REPORT_DIR @@ -182,7 +218,7 @@ end # Per System tasks desc "Run all django tests on our djangoapps for the #{system}" - task "test_#{system}", [:stop_on_failure] => ["clean_test_files", "#{system}:collectstatic:test", "fasttest_#{system}"] + task "test_#{system}", [:stop_on_failure] => ["clean_test_files", "#{system}:collect_assets:test", "fasttest_#{system}"] # Have a way to run the tests without running collectstatic -- useful when debugging without # messing with static files. @@ -201,6 +237,7 @@ end desc task system, [:env, :options] => [:predjango] do |t, args| args.with_defaults(:env => 'dev', :options => default_options[system]) + compile_assets(watch=true) sh(django_admin(system, args.env, 'runserver', args.options)) end @@ -213,8 +250,9 @@ end end desc "Run collectstatic in the specified environment" - task "#{system}:collectstatic:#{env}" => :predjango do - sh("#{django_admin(system, env, 'collectstatic', '--noinput')} > /tmp/collectstatic.out") do |ok, status| + task "#{system}:collect_assets:#{env}" do + compile_assets() + sh("#{django_admin(system, env, 'collectstatic', '--noinput')} > /dev/null") do |ok, status| if !ok abort "collectstatic failed!" end From afb60f790e6465603797c6555bed85b900213e58 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Tue, 16 Apr 2013 07:11:31 -0400 Subject: [PATCH 05/15] Change the name of the asset gathering task --- rakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rakefile b/rakefile index 7d26c027ae..6158fd1c77 100644 --- a/rakefile +++ b/rakefile @@ -249,8 +249,8 @@ end sh("echo 'import #{system}.envs.#{env}' | #{django_admin(system, env, 'shell')}") end - desc "Run collectstatic in the specified environment" - task "#{system}:collect_assets:#{env}" do + desc "Compile coffeescript and sass, and then run collectstatic in the specified environment" + task "#{system}:gather_assets:#{env}" do compile_assets() sh("#{django_admin(system, env, 'collectstatic', '--noinput')} > /dev/null") do |ok, status| if !ok From f80353121c36341f941cb9b0e8aab32fa1674ee9 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Tue, 16 Apr 2013 07:49:52 -0400 Subject: [PATCH 06/15] Don't clean pyc file and install xmodule before all arbitrary rake commands --- rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rakefile b/rakefile index 6158fd1c77..aa9b131396 100644 --- a/rakefile +++ b/rakefile @@ -378,7 +378,7 @@ end task :runserver => :lms desc "Run django-admin against the specified system and environment" -task "django-admin", [:action, :system, :env, :options] => [:predjango] do |t, args| +task "django-admin", [:action, :system, :env, :options] do |t, args| args.with_defaults(:env => 'dev', :system => 'lms', :options => '') sh(django_admin(args.system, args.env, args.action, args.options)) end From 6a36d9dba8ebe3e12c886a68661941b20201ad14 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Wed, 17 Apr 2013 09:41:04 -0400 Subject: [PATCH 07/15] Use the new gather_assets target before running tests --- rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rakefile b/rakefile index aa9b131396..3b96a2000e 100644 --- a/rakefile +++ b/rakefile @@ -218,7 +218,7 @@ end # Per System tasks desc "Run all django tests on our djangoapps for the #{system}" - task "test_#{system}", [:stop_on_failure] => ["clean_test_files", "#{system}:collect_assets:test", "fasttest_#{system}"] + task "test_#{system}", [:stop_on_failure] => ["clean_test_files", "#{system}:gather_assets:test", "fasttest_#{system}"] # Have a way to run the tests without running collectstatic -- useful when debugging without # messing with static files. From 3031c1da3809d15516468cbce1abb2c027ae8b76 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Thu, 18 Apr 2013 07:18:22 -0400 Subject: [PATCH 08/15] Fix up call to spawn --- rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rakefile b/rakefile index 3b96a2000e..87f1e78317 100644 --- a/rakefile +++ b/rakefile @@ -37,7 +37,7 @@ end # Runs Process.spawn, and kills the process at the end of the rake process # Expects the same arguments as Process.spawn def background_process(*command) - pid = Process.spawn(*command, :pgroup => true) + pid = Process.spawn({}, *command, {:pgroup => true}) at_exit do puts "Ending process and children" From bbab2d7de73138d5fc42908e25d633d29e29fdc3 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Thu, 18 Apr 2013 10:09:21 -0400 Subject: [PATCH 09/15] Install gemfiles for tests --- jenkins/test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins/test.sh b/jenkins/test.sh index b554e7a708..e1d44bf6b5 100755 --- a/jenkins/test.sh +++ b/jenkins/test.sh @@ -38,6 +38,8 @@ source /mnt/virtualenvs/"$JOB_NAME"/bin/activate pip install -q -r pre-requirements.txt yes w | pip install -q -r requirements.txt +bundle install + rake clobber rake pep8 > pep8.log || cat pep8.log rake pylint > pylint.log || cat pylint.log From 729daf5cbfb4dbf9a7b3b6c4fc915c2364f8b10c Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Thu, 18 Apr 2013 15:19:19 -0400 Subject: [PATCH 10/15] Move xmodule asset gathering out of server startup This allows us to a) compile the coffeescript and sass from xmodules using the new out-of-band method and b) reload xmodule static content whenever it changes, which should make devs much happier. --- cms/envs/common.py | 37 +++----------- cms/static/sass/base-style.scss | 4 +- cms/static/sass/bourbon | 1 - common/lib/xmodule/setup.py | 17 ++++-- common/lib/xmodule/xmodule/static_content.py | 54 +++++++++++++++----- lms/envs/common.py | 16 ++---- lms/static/sass/bourbon | 1 - lms/static/sass/course.scss | 2 +- rakefile | 36 ++++++++----- requirements.txt | 2 + 10 files changed, 91 insertions(+), 79 deletions(-) delete mode 120000 cms/static/sass/bourbon mode change 100644 => 100755 common/lib/xmodule/xmodule/static_content.py delete mode 120000 lms/static/sass/bourbon diff --git a/cms/envs/common.py b/cms/envs/common.py index 21995be40d..5ffb573a6c 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -20,11 +20,8 @@ Longer TODO: """ import sys -import os.path -import os import lms.envs.common from path import path -from xmodule.static_content import write_descriptor_styles, write_descriptor_js, write_module_js, write_module_styles ############################ FEATURE CONFIGURATION ############################# @@ -186,30 +183,7 @@ MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage' STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage' -# Load javascript and css from all of the available descriptors, and -# prep it for use in pipeline js -from xmodule.raw_module import RawDescriptor -from xmodule.error_module import ErrorDescriptor -from rooted_paths import rooted_glob, remove_root - -write_descriptor_styles(PROJECT_ROOT / "static/sass/descriptor", [RawDescriptor, ErrorDescriptor]) -write_module_styles(PROJECT_ROOT / "static/sass/module", [RawDescriptor, ErrorDescriptor]) - -descriptor_js = [path.replace('.coffee', '.js') for path in remove_root( - PROJECT_ROOT / 'static', - write_descriptor_js( - PROJECT_ROOT / "static/coffee/descriptor", - [RawDescriptor, ErrorDescriptor] - ) -)] - -module_js = [path.replace('.coffee', '.js') for path in remove_root( - PROJECT_ROOT / 'static', - write_module_js( - PROJECT_ROOT / "static/coffee/module", - [RawDescriptor, ErrorDescriptor] - ) -)] +from rooted_paths import rooted_glob PIPELINE_CSS = { 'base-style': { @@ -217,7 +191,9 @@ PIPELINE_CSS = { 'js/vendor/CodeMirror/codemirror.css', 'css/vendor/ui-lightness/jquery-ui-1.8.22.custom.css', 'css/vendor/jquery.qtip.min.css', - 'sass/base-style.css' + 'sass/base-style.css', + 'xmodule/modules.css', + 'xmodule/descriptor.css', ], 'output_filename': 'css/cms-base-style.css', }, @@ -232,7 +208,10 @@ PIPELINE_JS = { 'output_filename': 'js/cms-application.js', }, 'module-js': { - 'source_filenames': descriptor_js + module_js, + 'source_filenames': ( + rooted_glob(COMMON_ROOT / 'static/', 'xmodule/descriptors/js/*.js') + + rooted_glob(COMMON_ROOT / 'static/', 'xmodule/modules/js/*.js') + ), 'output_filename': 'js/cms-modules.js', }, 'spec': { diff --git a/cms/static/sass/base-style.scss b/cms/static/sass/base-style.scss index a355b3a03f..ee6ff18d43 100644 --- a/cms/static/sass/base-style.scss +++ b/cms/static/sass/base-style.scss @@ -54,5 +54,5 @@ @import 'assets/content-types'; // xblock-related -@import 'module/module-styles.scss'; -@import 'descriptor/module-styles.scss'; +@import 'xmodule/modules/css/module-styles.scss'; +@import 'xmodule/descriptors/css/module-styles.scss'; diff --git a/cms/static/sass/bourbon b/cms/static/sass/bourbon deleted file mode 120000 index 6f53a8b404..0000000000 --- a/cms/static/sass/bourbon +++ /dev/null @@ -1 +0,0 @@ -../../../common/static/sass/bourbon/ \ No newline at end of file diff --git a/common/lib/xmodule/setup.py b/common/lib/xmodule/setup.py index 85d42690b9..982a77631d 100644 --- a/common/lib/xmodule/setup.py +++ b/common/lib/xmodule/setup.py @@ -4,13 +4,15 @@ setup( name="XModule", version="0.1", packages=find_packages(exclude=["tests"]), - install_requires=['distribute'], + install_requires=[ + 'distribute', + 'docopt', + 'capa', + 'path.py', + ], package_data={ 'xmodule': ['js/module/*'] }, - requires=[ - 'capa', - ], # See http://guide.python-distribute.org/creation.html#entry-points # for a description of entry_points @@ -50,6 +52,11 @@ setup( "graphical_slider_tool = xmodule.gst_module:GraphicalSliderToolDescriptor", "annotatable = xmodule.annotatable_module:AnnotatableDescriptor", "foldit = xmodule.foldit_module:FolditDescriptor", - ] + "hidden = xmodule.hidden_module:HiddenDescriptor", + "raw = xmodule.raw_module:RawDescriptor", + ], + 'console_scripts': [ + 'xmodule_assets = xmodule.static_content:main', + ] } ) diff --git a/common/lib/xmodule/xmodule/static_content.py b/common/lib/xmodule/xmodule/static_content.py old mode 100644 new mode 100755 index ae9b55f313..a79f83cfe5 --- a/common/lib/xmodule/xmodule/static_content.py +++ b/common/lib/xmodule/xmodule/static_content.py @@ -1,3 +1,4 @@ +# /usr/bin/env python """ This module has utility functions for gathering up the static content that is defined by XModules and XModuleDescriptors (javascript and css) @@ -6,40 +7,43 @@ that is defined by XModules and XModuleDescriptors (javascript and css) import hashlib import os import errno +import sys from collections import defaultdict +from docopt import docopt +from path import path -from .x_module import XModuleDescriptor +from xmodule.x_module import XModuleDescriptor -def write_module_styles(output_root, extra_descriptors): - return _write_styles('.xmodule_display', output_root, _list_modules(extra_descriptors)) +def write_module_styles(output_root): + return _write_styles('.xmodule_display', output_root, _list_modules()) -def write_module_js(output_root, extra_descriptors): - return _write_js(output_root, _list_modules(extra_descriptors)) +def write_module_js(output_root): + return _write_js(output_root, _list_modules()) -def write_descriptor_styles(output_root, extra_descriptors): - return _write_styles('.xmodule_edit', output_root, _list_descriptors(extra_descriptors)) +def write_descriptor_styles(output_root): + return _write_styles('.xmodule_edit', output_root, _list_descriptors()) -def write_descriptor_js(output_root, extra_descriptors): - return _write_js(output_root, _list_descriptors(extra_descriptors)) +def write_descriptor_js(output_root): + return _write_js(output_root, _list_descriptors()) -def _list_descriptors(extra_descriptors): +def _list_descriptors(): return [ desc for desc in [ desc for (_, desc) in XModuleDescriptor.load_classes() - ] + extra_descriptors + ] ] -def _list_modules(extra_descriptors): +def _list_modules(): return [ desc.module_class for desc - in _list_descriptors(extra_descriptors) + in _list_descriptors() ] @@ -76,9 +80,12 @@ def _write_styles(selector, output_root, classes): css_imports[class_].add(fragment_name) with open(output_root / '_module-styles.scss', 'w') as module_styles: + + module_styles.write("@import 'bourbon/bourbon';\n") + module_styles.write("@import 'bourbon/addons/button';\n") for class_, fragment_names in css_imports.items(): imports = "\n".join('@import "{0}";'.format(name) for name in fragment_names) - module_styles.write("""{selector}.xmodule_{class_} {{ {imports} }}""".format( + module_styles.write("""{selector}.xmodule_{class_} {{ {imports} }}\n""".format( class_=class_, imports=imports, selector=selector )) @@ -105,3 +112,22 @@ def _write_js(output_root, classes): module_js.append(path) return module_js + + +def main(): + """ + Generate + Usage: static_content.py + """ + args = docopt(main.__doc__) + root = path(args['']) + + root.rmtree() + write_descriptor_js(root / 'descriptors/js') + write_descriptor_styles(root / 'descriptors/css') + write_module_js(root / 'modules/js') + write_module_styles(root / 'modules/css') + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/lms/envs/common.py b/lms/envs/common.py index 27c2cfc022..9b9e7897c6 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -20,7 +20,6 @@ Longer TODO: """ import sys import os -from xmodule.static_content import write_module_styles, write_module_js from path import path @@ -390,15 +389,7 @@ MIDDLEWARE_CLASSES = ( STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage' -from xmodule.hidden_module import HiddenDescriptor -from rooted_paths import rooted_glob, remove_root - -write_module_styles(PROJECT_ROOT / 'static/sass/module', [HiddenDescriptor]) - -module_js = [path.replace('.coffee', '.js') for path in remove_root( - PROJECT_ROOT / 'static', - write_module_js(PROJECT_ROOT / 'static/coffee/module', [HiddenDescriptor]) -)] +from rooted_paths import rooted_glob courseware_js = ( [ @@ -436,7 +427,8 @@ PIPELINE_CSS = { 'css/vendor/jquery.treeview.css', 'css/vendor/ui-lightness/jquery-ui-1.8.22.custom.css', 'css/vendor/jquery.qtip.min.css', - 'sass/course.css' + 'sass/course.css', + 'xmodule/modules.css', ], 'output_filename': 'css/lms-course.css', }, @@ -472,7 +464,7 @@ PIPELINE_JS = { 'output_filename': 'js/lms-main_vendor.js', }, 'module-js': { - 'source_filenames': module_js, + 'source_filenames': rooted_glob(COMMON_ROOT / 'static', 'xmodule/modules/js/*.js'), 'output_filename': 'js/lms-modules.js', }, 'discussion': { diff --git a/lms/static/sass/bourbon b/lms/static/sass/bourbon deleted file mode 120000 index 6f53a8b404..0000000000 --- a/lms/static/sass/bourbon +++ /dev/null @@ -1 +0,0 @@ -../../../common/static/sass/bourbon/ \ No newline at end of file diff --git a/lms/static/sass/course.scss b/lms/static/sass/course.scss index 60eda5a5c8..e57865fa9d 100644 --- a/lms/static/sass/course.scss +++ b/lms/static/sass/course.scss @@ -15,7 +15,7 @@ @import 'course/base/mixins'; @import 'course/base/base'; @import 'course/base/extends'; -@import 'module/module-styles.scss'; +@import 'xmodule/modules/css/module-styles.scss'; // courseware @import 'course/courseware/courseware'; diff --git a/rakefile b/rakefile index 87f1e78317..ecf8242276 100644 --- a/rakefile +++ b/rakefile @@ -120,21 +120,29 @@ def report_dir_path(dir) end def compile_assets(watch=false) - coffee_cmd = "coffee #{watch ? '--watch' : ''} --compile */static" - sass_cmd = "sass --style compressed --require ./common/static/sass/bourbon/lib/bourbon.rb #{watch ? '--watch' : '--update'} */static" - + xmodule_cmd = 'xmodule_assets common/static/xmodule' if watch - background_process(coffee_cmd) - background_process(sass_cmd) - else - coffee_pid = Process.spawn(coffee_cmd) - puts "Waiting for coffee to complete (pid #{coffee_pid})" - Process.wait(coffee_pid) - puts "Coffee completed" - sass_pid = Process.spawn(sass_cmd) - puts "Waiting for sass to complete (pid #{sass_pid})" - Process.wait(sass_pid) - puts "Sass completed" + xmodule_cmd = "watchmedo shell-command \ + --patterns='*.js;*.coffee;*.sass;*.scss;*.css' \ + --recursive \ + --command='#{xmodule_cmd}' \ + common/lib/xmodule" + end + coffee_cmd = "coffee #{watch ? '--watch' : ''} --compile */static" + sass_cmd = "sass --style compressed " + + "--load-path ./common/static/sass " + + "--require ./common/static/sass/bourbon/lib/bourbon.rb " + + "#{watch ? '--watch' : '--update'} */static" + + [xmodule_cmd, coffee_cmd, sass_cmd].each do |cmd| + if watch + background_process(cmd) + else + pid = Process.spawn(cmd) + puts "Waiting for `#{cmd}` to complete (pid #{pid})" + Process.wait(pid) + puts "Completed" + end end end diff --git a/requirements.txt b/requirements.txt index a626ac1944..2fed48def7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,6 +50,8 @@ xmltodict==0.4.1 # Used for debugging ipython==0.13.1 +# Used for development operation +watchdog==0.6.0 # Metrics gathering and monitoring dogapi==1.2.1 From f7cad77b93d9c3a45a537fa5163fd0f089220934 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Thu, 18 Apr 2013 15:45:32 -0400 Subject: [PATCH 11/15] Ignore errors when deleting the old xmodule compiled assets directory --- common/lib/xmodule/xmodule/static_content.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/xmodule/xmodule/static_content.py b/common/lib/xmodule/xmodule/static_content.py index a79f83cfe5..c110d310eb 100755 --- a/common/lib/xmodule/xmodule/static_content.py +++ b/common/lib/xmodule/xmodule/static_content.py @@ -122,7 +122,7 @@ def main(): args = docopt(main.__doc__) root = path(args['']) - root.rmtree() + root.rmtree(ignore_errors=True) write_descriptor_js(root / 'descriptors/js') write_descriptor_styles(root / 'descriptors/css') write_module_js(root / 'modules/js') From ec2532ac2e0a78b76386af31a7117021f8bcb767 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Thu, 18 Apr 2013 15:45:52 -0400 Subject: [PATCH 12/15] Before starting the server, forcibly rebuild assets before starting to watch them --- rakefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rakefile b/rakefile index ecf8242276..fa135844ff 100644 --- a/rakefile +++ b/rakefile @@ -132,7 +132,7 @@ def compile_assets(watch=false) sass_cmd = "sass --style compressed " + "--load-path ./common/static/sass " + "--require ./common/static/sass/bourbon/lib/bourbon.rb " + - "#{watch ? '--watch' : '--update'} */static" + "#{watch ? '--watch' : '--update --force'} */static" [xmodule_cmd, coffee_cmd, sass_cmd].each do |cmd| if watch @@ -245,6 +245,7 @@ end desc task system, [:env, :options] => [:predjango] do |t, args| args.with_defaults(:env => 'dev', :options => default_options[system]) + compile_assets(watch=false) compile_assets(watch=true) sh(django_admin(system, args.env, 'runserver', args.options)) end From 2ff4564686b3e7096e83101d4205ed328666fff0 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Thu, 18 Apr 2013 15:46:08 -0400 Subject: [PATCH 13/15] Ignore compiled xmodule assets --- common/static/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 common/static/.gitignore diff --git a/common/static/.gitignore b/common/static/.gitignore new file mode 100644 index 0000000000..f2c422d5b0 --- /dev/null +++ b/common/static/.gitignore @@ -0,0 +1 @@ +xmodule From 24f97d29db6d7c425dc78a375b43440d99b1b634 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Mon, 22 Apr 2013 08:02:09 -0400 Subject: [PATCH 14/15] Add sass sourcemaps back into the debug build --- package.json | 5 +++++ rakefile | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 package.json diff --git a/package.json b/package.json new file mode 100644 index 0000000000..4ce95d04ce --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "name": "mitx", + "version": "0.1.0", + "dependencies": { "coffee-script": "1.6.x"} +} \ No newline at end of file diff --git a/rakefile b/rakefile index fa135844ff..ae5dab2bb8 100644 --- a/rakefile +++ b/rakefile @@ -119,7 +119,7 @@ def report_dir_path(dir) return File.join(REPORT_DIR, dir.to_s) end -def compile_assets(watch=false) +def compile_assets(watch=false, debug=false) xmodule_cmd = 'xmodule_assets common/static/xmodule' if watch xmodule_cmd = "watchmedo shell-command \ @@ -129,7 +129,7 @@ def compile_assets(watch=false) common/lib/xmodule" end coffee_cmd = "coffee #{watch ? '--watch' : ''} --compile */static" - sass_cmd = "sass --style compressed " + + sass_cmd = "sass #{debug ? '--debug-info' : '--style compressed'} " + "--load-path ./common/static/sass " + "--require ./common/static/sass/bourbon/lib/bourbon.rb " + "#{watch ? '--watch' : '--update --force'} */static" @@ -245,8 +245,8 @@ end desc task system, [:env, :options] => [:predjango] do |t, args| args.with_defaults(:env => 'dev', :options => default_options[system]) - compile_assets(watch=false) - compile_assets(watch=true) + compile_assets(watch=false, debug=true) + compile_assets(watch=true, debug=true) sh(django_admin(system, args.env, 'runserver', args.options)) end From 4d55f87ab137975dc7cdf09291eebb56fe46b3e2 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Wed, 24 Apr 2013 12:33:44 -0400 Subject: [PATCH 15/15] Add comments on duplicate calls to compile_assets --- rakefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rakefile b/rakefile index ae5dab2bb8..dc123db3f8 100644 --- a/rakefile +++ b/rakefile @@ -245,8 +245,13 @@ end desc task system, [:env, :options] => [:predjango] do |t, args| args.with_defaults(:env => 'dev', :options => default_options[system]) + + # Compile all assets first compile_assets(watch=false, debug=true) + + # Listen for any changes to assets compile_assets(watch=true, debug=true) + sh(django_admin(system, args.env, 'runserver', args.options)) end