Merge pull request #5 from edx/nate/asset-preprocessing-mgmt-cmd

Preprocess assets as Django management command

cms/envs/common.py

@@ -323,6 +323,7 @@ INSTALLED_APPS = (
     'track',
 
     # For asset pipelining
+    'mitxmako',
     'pipeline',
     'staticfiles',
     'static_replace',

common/djangoapps/mitxmako/management/commands/preprocess_assets.py

@@ -0,0 +1,65 @@
+"""
+Preprocess templatized asset files, enabling asset authors to use
+Python/Django inside of Sass and CoffeeScript. This preprocessing
+will happen before the invocation of the asset compiler (currently
+handled by the asset Rakefile).
+
+For this to work, assets need to be named with the appropriate
+template extension (e.g., .mako for Mako templates). Currently Mako
+is the only template engine supported.
+"""
+import os
+
+from django.core.management.base import NoArgsCommand
+from django.conf import settings
+
+from mako.template import Template
+
+class Command(NoArgsCommand):
+    """
+    Basic management command to preprocess asset template files.
+    """
+
+    help = "Preprocess asset template files to ready them for compilation."
+
+    def handle_noargs(self, **options):
+        """
+        Walk over all of the static files directories specified in the
+        settings file, looking for asset template files (indicated by
+        a file extension like .mako).
+        """
+        for staticfiles_dir in getattr(settings, "STATICFILES_DIRS", []):
+            # Cribbed from the django-staticfiles app at:
+            # https://github.com/jezdez/django-staticfiles/blob/develop/staticfiles/finders.py#L52
+            if isinstance(staticfiles_dir, (list, tuple)):
+                prefix, staticfiles_dir = staticfiles_dir
+
+            # Walk over the current static files directory tree,
+            # preprocessing files that have a template extension.
+            for root, dirs, files in os.walk(staticfiles_dir):
+                for filename in files:
+                    outfile, extension = os.path.splitext(filename)
+                    # We currently only handle Mako templates
+                    if extension == ".mako":
+                        self.__preprocess(os.path.join(root, filename),
+                                          os.path.join(root, outfile))
+
+
+    def __context(self):
+        """
+        Return a dict that contains all of the available context
+        variables to the asset template.
+        """
+        # TODO: do we need to include anything else?
+        # TODO: do this with the django-settings-context-processor
+        return { "THEME_NAME" : getattr(settings, "THEME_NAME", None) }
+
+
+    def __preprocess(self, infile, outfile):
+        """
+        Run `infile` through the Mako template engine, storing the
+        result in `outfile`.
+        """
+        with open(outfile, "w") as _outfile:
+            _outfile.write(Template(filename=str(infile)).render(env=self.__context()))
+

lms/envs/common.py

@@ -662,6 +662,7 @@ INSTALLED_APPS = (
     'service_status',
 
     # For asset pipelining
+    'mitxmako',
     'pipeline',
     'staticfiles',
     'static_replace',

rakefiles/assets.rake

@@ -6,30 +6,6 @@ if USE_CUSTOM_THEME
     THEME_SASS = File.join(THEME_ROOT, "static", "sass")
 end
 
-# Run the specified file through the Mako templating engine, providing
-# the ENV_TOKENS to the templating context.
-def preprocess_with_mako(filename)
-    # simple command-line invocation of Mako engine
-    # cdodge: the .gsub() are used to translate true->True and false->False to make the generated
-    # python actually valid python. This is just a short term hack to unblock the release train
-    # until a real fix can be made by people who know this better
-    mako = "from mako.template import Template;" +
-           "print Template(filename=\"#{filename}\")" +
-           # Total hack. It works because a Python dict literal has
-           # the same format as a JSON object.
-           ".render(env=#{ENV_TOKENS.to_json.gsub("true","True").gsub("false","False")});"
-
-    # strip off the .mako extension
-    output_filename = filename.chomp(File.extname(filename))
-
-    # just pipe from stdout into the new file, exiting on failure
-    File.open(output_filename, 'w') do |file|
-      file.write(`python -c '#{mako}'`)
-      exit_code = $?.to_i
-      abort "#{mako} failed with #{exit_code}" if exit_code.to_i != 0
-    end
-end
-
 def xmodule_cmd(watch=false, debug=false)
     xmodule_cmd = 'xmodule_assets common/static/xmodule'
     if watch
@@ -84,11 +60,12 @@ namespace :assets do
     desc "Compile all assets in debug mode"
     multitask :debug
 
-    desc "Preprocess all static assets that have the .mako extension"
-    task :preprocess do
-      # Run assets through the Mako templating engine. Right now we
-      # just hardcode the asset filenames.
-      preprocess_with_mako("lms/static/sass/application.scss.mako")
+    desc "Preprocess all templatized static asset files"
+    task :preprocess, [:system, :env] do |t, args|
+      args.with_defaults(:system => "lms", :env => "dev")
+      sh(django_admin(args.system, args.env, "preprocess_assets")) do |ok, status|
+        abort "asset preprocessing failed!" if !ok
+      end
     end
 
     desc "Watch all assets for changes and automatically recompile"
@@ -138,7 +115,6 @@ namespace :assets do
         end
     end
 
-
     multitask :sass => 'assets:xmodule'
     namespace :sass do
         # In watch mode, sass doesn't immediately compile out of date files,
@@ -153,16 +129,25 @@ namespace :assets do
     end
 end
 
-[:lms, :cms].each do |system|
-    # Per environment tasks
-    environments(system).each do |env|
-        desc "Compile coffeescript and sass, and then run collectstatic in the specified environment"
-        task "#{system}:gather_assets:#{env}" => :assets do
-            sh("#{django_admin(system, env, 'collectstatic', '--noinput')} > /dev/null") do |ok, status|
-                if !ok
-                    abort "collectstatic failed!"
-                end
-            end
+# This task does the real heavy lifting to gather all of the static
+# assets. We want people to call it via the wrapper below, so we
+# don't provide a description so that it won't show up in rake -T.
+task :gather_assets, [:system, :env] => :assets do |t, args|
+    sh("#{django_admin(args.system, args.env, 'collectstatic', '--noinput')} > /dev/null") do |ok, status|
+        if !ok
+            abort "collectstatic failed!"
+        end
+    end
+end
+
+[:lms, :cms].each do |system|
+    # Per environment tasks
+    environments(system).each do |env|
+        # This task wraps the one above, since we need the system and
+        # env arguments to be passed to all dependent tasks.
+        desc "Compile coffeescript and sass, and then run collectstatic in the specified environment"
+        task "#{system}:gather_assets:#{env}" do
+          Rake::Task[:gather_assets].invoke(system, env)
         end
     end
 end

rakefiles/django.rake

@@ -15,14 +15,22 @@ task :fastlms do
     sh("#{django_admin} runserver --traceback --settings=lms.envs.dev   --pythonpath=.")
 end
 
+# Start :system locally with the specified :env and :options.
+#
+# This task should be invoked via the wrapper below, so we don't
+# include a description to keep it from showing up in rake -T.
+task :runserver, [:system, :env, :options] => [:install_prereqs, 'assets:_watch', :predjango] do |t, args|
+    sh(django_admin(args.system, args.env, 'runserver', args.options))
+end
+
 [:lms, :cms].each do |system|
     desc <<-desc
         Start the #{system} locally with the specified environment (defaults to dev).
         Other useful environments are devplus (for dev testing with a real local database)
         desc
-    task system, [:env, :options] => [:install_prereqs, 'assets:_watch', :predjango] do |t, args|
+    task system, [:env, :options] do |t, args|
         args.with_defaults(:env => 'dev', :options => default_options[system])
-        sh(django_admin(system, args.env, 'runserver', args.options))
+        Rake::Task[:runserver].invoke(system, args.env, args.options)
     end
 
     desc "Start #{system} Celery worker"

rakefiles/jasmine.rake

@@ -73,21 +73,43 @@ def run_phantom_js(url)
     sh("#{phantomjs} node_modules/jasmine-reporters/test/phantomjs-testrunner.js #{url}")
 end
 
+# Open jasmine tests for :system in the default browser. The :env
+# should (always?) be 'jasmine', but it's passed as an arg so that
+# the :assets dependency gets it.
+#
+# This task should be invoked via the wrapper below, so we don't
+# include a description to keep it from showing up in rake -T.
+task :browse_jasmine, [:system, :env] => :assets do |t, args|
+    django_for_jasmine(args.system, true) do |jasmine_url|
+        Launchy.open(jasmine_url)
+        puts "Press ENTER to terminate".red
+        $stdin.gets
+    end
+end
+
+# Use phantomjs to run jasmine tests from the console. The :env
+# should (always?) be 'jasmine', but it's passed as an arg so that
+# the :assets dependency gets it.
+#
+# This task should be invoked via the wrapper below, so we don't
+# include a description to keep it from showing up in rake -T.
+task :phantomjs_jasmine, [:system, :env] => :assets do |t, args|
+    django_for_jasmine(args.system, false) do |jasmine_url|
+        run_phantom_js(jasmine_url)
+    end
+end
+
+# Wrapper tasks for the real browse_jasmine and phantomjs_jasmine
+# tasks above. These have a nicer UI since there's no arg passing.
 [:lms, :cms].each do |system|
     desc "Open jasmine tests for #{system} in your default browser"
-    task "browse_jasmine_#{system}" => :assets do
-        django_for_jasmine(system, true) do |jasmine_url|
-            Launchy.open(jasmine_url)
-            puts "Press ENTER to terminate".red
-            $stdin.gets
-        end
+    task "browse_jasmine_#{system}" do
+        Rake::Task[:browse_jasmine].invoke(system, 'jasmine')
     end
 
     desc "Use phantomjs to run jasmine tests for #{system} from the console"
-    task "phantomjs_jasmine_#{system}" => :assets do
-        django_for_jasmine(system, false) do |jasmine_url|
-            run_phantom_js(jasmine_url)
-        end
+    task "phantomjs_jasmine_#{system}" do
+        Rake::Task[:phantomjs_jasmine].invoke(system, 'jasmine')
     end
 end