Move xmodule asset gathering out of server startup

This allows us to a) compile the coffeescript and sass from xmodules
using the new out-of-band method and b) reload xmodule static content
whenever it changes, which should make devs much happier.

cms/envs/common.py

@@ -20,11 +20,8 @@ Longer TODO:
 """
 
 import sys
-import os.path
-import os
 import lms.envs.common
 from path import path
-from xmodule.static_content import write_descriptor_styles, write_descriptor_js, write_module_js, write_module_styles
 
 ############################ FEATURE CONFIGURATION #############################
 
@@ -186,30 +183,7 @@ MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage'
 
 STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'
 
-# Load javascript and css from all of the available descriptors, and
-# prep it for use in pipeline js
-from xmodule.raw_module import RawDescriptor
-from xmodule.error_module import ErrorDescriptor
-from rooted_paths import rooted_glob, remove_root
-
-write_descriptor_styles(PROJECT_ROOT / "static/sass/descriptor", [RawDescriptor, ErrorDescriptor])
-write_module_styles(PROJECT_ROOT / "static/sass/module", [RawDescriptor, ErrorDescriptor])
-
-descriptor_js = [path.replace('.coffee', '.js') for path in remove_root(
-    PROJECT_ROOT / 'static',
-    write_descriptor_js(
-        PROJECT_ROOT / "static/coffee/descriptor",
-        [RawDescriptor, ErrorDescriptor]
-    )
-)]
-
-module_js = [path.replace('.coffee', '.js') for path in remove_root(
-    PROJECT_ROOT / 'static',
-    write_module_js(
-        PROJECT_ROOT / "static/coffee/module",
-        [RawDescriptor, ErrorDescriptor]
-    )
-)]
+from rooted_paths import rooted_glob
 
 PIPELINE_CSS = {
     'base-style': {
@@ -217,7 +191,9 @@ PIPELINE_CSS = {
             'js/vendor/CodeMirror/codemirror.css',
             'css/vendor/ui-lightness/jquery-ui-1.8.22.custom.css',
             'css/vendor/jquery.qtip.min.css',
-            'sass/base-style.css'
+            'sass/base-style.css',
+            'xmodule/modules.css',
+            'xmodule/descriptor.css',
         ],
         'output_filename': 'css/cms-base-style.css',
     },
@@ -232,7 +208,10 @@ PIPELINE_JS = {
         'output_filename': 'js/cms-application.js',
     },
     'module-js': {
-        'source_filenames': descriptor_js + module_js,
+        'source_filenames': (
+            rooted_glob(COMMON_ROOT / 'static/', 'xmodule/descriptors/js/*.js') +
+            rooted_glob(COMMON_ROOT / 'static/', 'xmodule/modules/js/*.js')
+        ),
         'output_filename': 'js/cms-modules.js',
     },
     'spec': {

cms/static/sass/base-style.scss

@@ -54,5 +54,5 @@
 @import 'assets/content-types';
 
 // xblock-related
-@import 'module/module-styles.scss';
-@import 'descriptor/module-styles.scss';
+@import 'xmodule/modules/css/module-styles.scss';
+@import 'xmodule/descriptors/css/module-styles.scss';

cms/static/sass/bourbon

@@ -1 +0,0 @@
-../../../common/static/sass/bourbon/

common/lib/xmodule/setup.py

@@ -4,13 +4,15 @@ setup(
     name="XModule",
     version="0.1",
     packages=find_packages(exclude=["tests"]),
-    install_requires=['distribute'],
+    install_requires=[
+        'distribute',
+        'docopt',
+        'capa',
+        'path.py',
+    ],
     package_data={
         'xmodule': ['js/module/*']
     },
-    requires=[
-        'capa',
-    ],
 
     # See http://guide.python-distribute.org/creation.html#entry-points
     # for a description of entry_points
@@ -50,6 +52,11 @@ setup(
             "graphical_slider_tool = xmodule.gst_module:GraphicalSliderToolDescriptor",
             "annotatable = xmodule.annotatable_module:AnnotatableDescriptor",
             "foldit = xmodule.foldit_module:FolditDescriptor",
-         ]
+            "hidden = xmodule.hidden_module:HiddenDescriptor",
+            "raw = xmodule.raw_module:RawDescriptor",
+        ],
+        'console_scripts': [
+            'xmodule_assets = xmodule.static_content:main',
+        ]
     }
 )

common/lib/xmodule/xmodule/static_content.py

@@ -1,3 +1,4 @@
+# /usr/bin/env python
 """
 This module has utility functions for gathering up the static content
 that is defined by XModules and XModuleDescriptors (javascript and css)
@@ -6,40 +7,43 @@ that is defined by XModules and XModuleDescriptors (javascript and css)
 import hashlib
 import os
 import errno
+import sys
 from collections import defaultdict
+from docopt import docopt
+from path import path
 
-from .x_module import XModuleDescriptor
+from xmodule.x_module import XModuleDescriptor
 
 
-def write_module_styles(output_root, extra_descriptors):
-    return _write_styles('.xmodule_display', output_root, _list_modules(extra_descriptors))
+def write_module_styles(output_root):
+    return _write_styles('.xmodule_display', output_root, _list_modules())
 
 
-def write_module_js(output_root, extra_descriptors):
-    return _write_js(output_root, _list_modules(extra_descriptors))
+def write_module_js(output_root):
+    return _write_js(output_root, _list_modules())
 
 
-def write_descriptor_styles(output_root, extra_descriptors):
-    return _write_styles('.xmodule_edit', output_root, _list_descriptors(extra_descriptors))
+def write_descriptor_styles(output_root):
+    return _write_styles('.xmodule_edit', output_root, _list_descriptors())
 
 
-def write_descriptor_js(output_root, extra_descriptors):
-    return _write_js(output_root, _list_descriptors(extra_descriptors))
+def write_descriptor_js(output_root):
+    return _write_js(output_root, _list_descriptors())
 
 
-def _list_descriptors(extra_descriptors):
+def _list_descriptors():
     return [
         desc for desc in [
             desc for (_, desc) in XModuleDescriptor.load_classes()
-        ] + extra_descriptors
+        ]
     ]
 
 
-def _list_modules(extra_descriptors):
+def _list_modules():
     return [
         desc.module_class
         for desc
-        in _list_descriptors(extra_descriptors)
+        in _list_descriptors()
     ]
 
 
@@ -76,9 +80,12 @@ def _write_styles(selector, output_root, classes):
             css_imports[class_].add(fragment_name)
 
     with open(output_root / '_module-styles.scss', 'w') as module_styles:
+
+        module_styles.write("@import 'bourbon/bourbon';\n")
+        module_styles.write("@import 'bourbon/addons/button';\n")
         for class_, fragment_names in css_imports.items():
             imports = "\n".join('@import "{0}";'.format(name) for name in fragment_names)
-            module_styles.write("""{selector}.xmodule_{class_} {{ {imports} }}""".format(
+            module_styles.write("""{selector}.xmodule_{class_} {{ {imports} }}\n""".format(
                 class_=class_, imports=imports, selector=selector
             ))
 
@@ -105,3 +112,22 @@ def _write_js(output_root, classes):
         module_js.append(path)
 
     return module_js
+
+
+def main():
+    """
+    Generate
+    Usage: static_content.py <output_root>
+    """
+    args = docopt(main.__doc__)
+    root = path(args['<output_root>'])
+
+    root.rmtree()
+    write_descriptor_js(root / 'descriptors/js')
+    write_descriptor_styles(root / 'descriptors/css')
+    write_module_js(root / 'modules/js')
+    write_module_styles(root / 'modules/css')
+
+
+if __name__ == '__main__':
+    sys.exit(main())

lms/envs/common.py

@@ -20,7 +20,6 @@ Longer TODO:
 """
 import sys
 import os
-from xmodule.static_content import write_module_styles, write_module_js
 
 from path import path
 
@@ -390,15 +389,7 @@ MIDDLEWARE_CLASSES = (
 
 STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'
 
-from xmodule.hidden_module import HiddenDescriptor
-from rooted_paths import rooted_glob, remove_root
-
-write_module_styles(PROJECT_ROOT / 'static/sass/module', [HiddenDescriptor])
-
-module_js = [path.replace('.coffee', '.js') for path in remove_root(
-    PROJECT_ROOT / 'static',
-    write_module_js(PROJECT_ROOT / 'static/coffee/module', [HiddenDescriptor])
-)]
+from rooted_paths import rooted_glob
 
 courseware_js = (
     [
@@ -436,7 +427,8 @@ PIPELINE_CSS = {
             'css/vendor/jquery.treeview.css',
             'css/vendor/ui-lightness/jquery-ui-1.8.22.custom.css',
             'css/vendor/jquery.qtip.min.css',
-            'sass/course.css'
+            'sass/course.css',
+            'xmodule/modules.css',
         ],
         'output_filename': 'css/lms-course.css',
     },
@@ -472,7 +464,7 @@ PIPELINE_JS = {
         'output_filename': 'js/lms-main_vendor.js',
     },
     'module-js': {
-        'source_filenames': module_js,
+        'source_filenames': rooted_glob(COMMON_ROOT / 'static', 'xmodule/modules/js/*.js'),
         'output_filename': 'js/lms-modules.js',
     },
     'discussion': {

lms/static/sass/bourbon

@@ -1 +0,0 @@
-../../../common/static/sass/bourbon/

lms/static/sass/course.scss

@@ -15,7 +15,7 @@
 @import 'course/base/mixins';
 @import 'course/base/base';
 @import 'course/base/extends';
-@import 'module/module-styles.scss';
+@import 'xmodule/modules/css/module-styles.scss';
 
 // courseware
 @import 'course/courseware/courseware';

rakefile

@@ -120,21 +120,29 @@ def report_dir_path(dir)
 end
 
 def compile_assets(watch=false)
-    coffee_cmd = "coffee #{watch ? '--watch' : ''} --compile */static"
-    sass_cmd = "sass --style compressed --require ./common/static/sass/bourbon/lib/bourbon.rb #{watch ? '--watch' : '--update'} */static"
-
+    xmodule_cmd = 'xmodule_assets common/static/xmodule'
     if watch
-        background_process(coffee_cmd)
-        background_process(sass_cmd)
-    else
-        coffee_pid = Process.spawn(coffee_cmd)
-        puts "Waiting for coffee to complete (pid #{coffee_pid})"
-        Process.wait(coffee_pid)
-        puts "Coffee completed"
-        sass_pid = Process.spawn(sass_cmd)
-        puts "Waiting for sass to complete (pid #{sass_pid})"
-        Process.wait(sass_pid)
-        puts "Sass completed"
+        xmodule_cmd = "watchmedo shell-command \
+                       --patterns='*.js;*.coffee;*.sass;*.scss;*.css' \
+                       --recursive \
+                       --command='#{xmodule_cmd}' \
+                       common/lib/xmodule"
+    end
+    coffee_cmd = "coffee #{watch ? '--watch' : ''} --compile */static"
+    sass_cmd = "sass --style compressed " +
+                     "--load-path ./common/static/sass " +
+                     "--require ./common/static/sass/bourbon/lib/bourbon.rb " +
+                     "#{watch ? '--watch' : '--update'} */static"
+
+    [xmodule_cmd, coffee_cmd, sass_cmd].each do |cmd|
+        if watch
+            background_process(cmd)
+        else
+            pid = Process.spawn(cmd)
+            puts "Waiting for `#{cmd}` to complete (pid #{pid})"
+            Process.wait(pid)
+            puts "Completed"
+        end
     end
 end
 

requirements.txt

@@ -50,6 +50,8 @@ xmltodict==0.4.1
 # Used for debugging
 ipython==0.13.1
 
+# Used for development operation
+watchdog==0.6.0
 
 # Metrics gathering and monitoring
 dogapi==1.2.1