Merge pull request #174 from edx/cale/stabilize-dev-env

Cale/stabilize dev env

CHANGELOG.rst

@@ -5,6 +5,19 @@ These are notable changes in edx-platform.  This is a rolling list of changes,
 in roughly chronological order, most recent first.  Add your entries at or near
 the top.  Include a label indicating the component affected.
 
+XModule: Only write out assets files if the contents have changed.
+
+XModule: Don't delete generated xmodule asset files when compiling (for
+instance, when XModule provides a coffeescript file, don't delete
+the associated javascript)
+
+Common: Make asset watchers run as singletons (so they won't start if the
+watcher is already running in another shell).
+
+Common: Use coffee directly when watching for coffeescript file changes.
+
+Common: Make rake provide better error messages if packages are missing.
+
 Common: Repairs development documentation generation by sphinx.
 
 LMS: Problem rescoring.  Added options on the Grades tab of the

Gemfile

@@ -4,3 +4,4 @@ gem 'sass', '3.1.15'
 gem 'bourbon', '~> 1.3.6'
 gem 'colorize', '~> 0.5.8'
 gem 'launchy', '~> 2.1.2'
+gem 'sys-proctable', '~> 0.9.3'

common/lib/xmodule/xmodule/static_content.py

@@ -4,6 +4,7 @@ This module has utility functions for gathering up the static content
 that is defined by XModules and XModuleDescriptors (javascript and css)
 """
 
+import logging
 import hashlib
 import os
 import errno
@@ -15,6 +16,9 @@ from path import path
 from xmodule.x_module import XModuleDescriptor
 
 
+LOG = logging.getLogger(__name__)
+
+
 def write_module_styles(output_root):
     return _write_styles('.xmodule_display', output_root, _list_modules())
 
@@ -121,18 +125,32 @@ def _write_js(output_root, classes):
             type=filetype)
         contents[filename] = fragment
 
-    _write_files(output_root, contents)
+    _write_files(output_root, contents, {'.coffee': '.js'})
 
     return [output_root / filename for filename in contents.keys()]
 
 
-def _write_files(output_root, contents):
+def _write_files(output_root, contents, generated_suffix_map=None):
     _ensure_dir(output_root)
-    for extra_file in set(output_root.files()) - set(contents.keys()):
-        extra_file.remove_p()
+    to_delete = set(file.basename() for file in output_root.files()) - set(contents.keys())
+
+    if generated_suffix_map:
+        for output_file in contents.keys():
+            for suffix, generated_suffix in generated_suffix_map.items():
+                if output_file.endswith(suffix):
+                    to_delete.discard(output_file.replace(suffix, generated_suffix))
+
+    for extra_file in to_delete:
+        (output_root / extra_file).remove_p()
 
     for filename, file_content in contents.iteritems():
-        (output_root / filename).write_bytes(file_content)
+        output_file = output_root / filename
+
+        if not output_file.isfile() or output_file.read_md5() != hashlib.md5(file_content).digest():
+            LOG.debug("Writing %s", output_file)
+            output_file.write_bytes(file_content)
+        else:
+            LOG.debug("%s unchanged, skipping", output_file)
 
 
 def main():

doc/development.md

@@ -63,6 +63,25 @@ To get a full list of available rake tasks, use:
 
     rake -T
 
+### Troubleshooting
+
+#### Reference Error: XModule is not defined (javascript)
+This means that the javascript defining an xmodule hasn't loaded correctly. There are a number
+of different things that could be causing this:
+
+1. See `Error: watch EMFILE`
+
+#### Error: watch EMFILE (coffee)
+When running a development server, we also start a watcher process alongside to recompile coffeescript
+and sass as changes are made. On Mac OSX systems, the coffee watcher process takes more file handles
+than are allowed by default. This will result in `EMFILE` errors when coffeescript is running, and
+will prevent javascript from compiling, leading to the error 'XModule is not defined'
+
+To work around this issue, we use `Process::setrlimit` to set the number of allowed open files.
+Coffee watches both directories and files, so you will need to set this fairly high (anecdotally,
+8000 seems to do the trick on OSX 10.7.5, 10.8.3, and 10.8.4)
+
+
 ## Running Tests
 
 See `testing.md` for instructions on running the test suite.

rakefile

@@ -1,9 +1,11 @@
-require 'json'
-require 'rake/clean'
-require './rakefiles/helpers.rb'
-
-Dir['rakefiles/*.rake'].each do |rakefile|
-    import rakefile
+begin
+    require 'json'
+    require 'rake/clean'
+    require './rakelib/helpers.rb'
+rescue LoadError => error
+    puts "Import faild (#{error})"
+    puts "Please run `bundle install` to bootstrap ruby dependencies"
+    exit 1
 end
 
 # Build Constants

rakelib/assets.rake

@@ -6,6 +6,8 @@ if USE_CUSTOM_THEME
     THEME_SASS = File.join(THEME_ROOT, "static", "sass")
 end
 
+MINIMAL_DARWIN_NOFILE_LIMIT = 8000
+
 def xmodule_cmd(watch=false, debug=false)
     xmodule_cmd = 'xmodule_assets common/static/xmodule'
     if watch
@@ -21,24 +23,14 @@ def xmodule_cmd(watch=false, debug=false)
 end
 
 def coffee_cmd(watch=false, debug=false)
-    if watch
-        # On OSx, coffee fails with EMFILE when
-        # trying to watch all of our coffee files at the same
-        # time.
-        #
-        # Ref: https://github.com/joyent/node/issues/2479
-        #
-        # So, instead, we use watchmedo, which works around the problem
-        "watchmedo shell-command " +
-                  "--command 'node_modules/.bin/coffee -c ${watch_src_path}' " +
-                  "--recursive " +
-                  "--patterns '*.coffee' " +
-                  "--ignore-directories " +
-                  "--wait " +
-                  "."
-    else
-        'node_modules/.bin/coffee --compile .'
+    if watch && Launchy::Application.new.host_os_family.darwin?
+        available_files = Process::getrlimit(:NOFILE)[0]
+        if available_files < MINIMAL_DARWIN_NOFILE_LIMIT
+            Process.setrlimit(:NOFILE, MINIMAL_DARWIN_NOFILE_LIMIT)
+
+        end
     end
+    "node_modules/.bin/coffee --compile #{watch ? '--watch' : ''} ."
 end
 
 def sass_cmd(watch=false, debug=false)
@@ -55,8 +47,9 @@ def sass_cmd(watch=false, debug=false)
           "#{watch ? '--watch' : '--update'} -E utf-8 #{sass_watch_paths.join(' ')}"
 end
 
+# This task takes arguments purely to pass them via dependencies to the preprocess task
 desc "Compile all assets"
-multitask :assets => 'assets:all'
+task :assets, [:system, :env] => 'assets:all'
 
 namespace :assets do
 
@@ -80,8 +73,9 @@ namespace :assets do
     {:xmodule => [:install_python_prereqs],
      :coffee => [:install_node_prereqs, :'assets:coffee:clobber'],
      :sass => [:install_ruby_prereqs, :preprocess]}.each_pair do |asset_type, prereq_tasks|
+        # This task takes arguments purely to pass them via dependencies to the preprocess task
         desc "Compile all #{asset_type} assets"
-        task asset_type => prereq_tasks do
+        task asset_type, [:system, :env] => prereq_tasks do |t, args|
             cmd = send(asset_type.to_s + "_cmd", watch=false, debug=false)
             if cmd.kind_of?(Array)
                 cmd.each {|c| sh(c)}
@@ -90,7 +84,8 @@ namespace :assets do
             end
         end
 
-        multitask :all => asset_type
+        # This task takes arguments purely to pass them via dependencies to the preprocess task
+        multitask :all, [:system, :env] => asset_type
         multitask :debug => "assets:#{asset_type}:debug"
         multitask :_watch => "assets:#{asset_type}:_watch"
 
@@ -111,9 +106,9 @@ namespace :assets do
             task :_watch => (prereq_tasks + ["assets:#{asset_type}:debug"]) do
                 cmd = send(asset_type.to_s + "_cmd", watch=true, debug=true)
                 if cmd.kind_of?(Array)
-                    cmd.each {|c| background_process(c)}
+                    cmd.each {|c| singleton_process(c)}
                 else
-                    background_process(cmd)
+                    singleton_process(cmd)
                 end
             end
         end

rakelib/helpers.rb

@@ -1,4 +1,6 @@
 require 'digest/md5'
+require 'sys/proctable'
+require 'colorize'
 
 def find_executable(exec)
     path = %x(which #{exec}).strip
@@ -84,6 +86,16 @@ def background_process(*command)
     end
 end
 
+# Runs a command as a background process, as long as no other processes
+# tagged with the same tag are running
+def singleton_process(*command)
+    if Sys::ProcTable.ps.select {|proc| proc.cmdline.include?(command.join(' '))}.empty?
+        background_process(*command)
+    else
+        puts "Process '#{command.join(' ')} already running, skipping".blue
+    end
+end
+
 def environments(system)
     Dir["#{system}/envs/**/*.py"].select{|file| ! (/__init__.py$/ =~ file)}.map do |env_file|
         env_file.gsub("#{system}/envs/", '').gsub(/\.py/, '').gsub('/', '.')

rakelib/prereqs.rake

@@ -1,5 +1,3 @@
-require './rakefiles/helpers.rb'
-
 PREREQS_MD5_DIR = ENV["PREREQ_CACHE_DIR"] || File.join(REPO_ROOT, '.prereqs_cache')
 
 CLOBBER.include(PREREQS_MD5_DIR)