520 lines
17 KiB
Plaintext
520 lines
17 KiB
Plaintext
require 'rake/clean'
|
|
require 'tempfile'
|
|
require 'net/http'
|
|
require 'launchy'
|
|
require 'colorize'
|
|
require 'erb'
|
|
require 'tempfile'
|
|
|
|
# Build Constants
|
|
REPO_ROOT = File.dirname(__FILE__)
|
|
BUILD_DIR = File.join(REPO_ROOT, "build")
|
|
REPORT_DIR = File.join(REPO_ROOT, "reports")
|
|
LMS_REPORT_DIR = File.join(REPORT_DIR, "lms")
|
|
|
|
# Packaging constants
|
|
DEPLOY_DIR = "/opt/wwc"
|
|
PACKAGE_NAME = "mitx"
|
|
LINK_PATH = "/opt/wwc/mitx"
|
|
PKG_VERSION = "0.1"
|
|
COMMIT = (ENV["GIT_COMMIT"] || `git rev-parse HEAD`).chomp()[0, 10]
|
|
BRANCH = (ENV["GIT_BRANCH"] || `git symbolic-ref -q HEAD`).chomp().gsub('refs/heads/', '').gsub('origin/', '')
|
|
BUILD_NUMBER = (ENV["BUILD_NUMBER"] || "dev").chomp()
|
|
|
|
# Set up the clean and clobber tasks
|
|
CLOBBER.include(BUILD_DIR, REPORT_DIR, 'test_root/*_repo', 'test_root/staticfiles')
|
|
CLEAN.include("#{BUILD_DIR}/*.deb", "#{BUILD_DIR}/util")
|
|
|
|
def select_executable(*cmds)
|
|
cmds.find_all{ |cmd| system("which #{cmd} > /dev/null 2>&1") }[0] || fail("No executables found from #{cmds.join(', ')}")
|
|
end
|
|
|
|
def django_admin(system, env, command, *args)
|
|
django_admin = ENV['DJANGO_ADMIN_PATH'] || select_executable('django-admin.py', 'django-admin')
|
|
return "#{django_admin} #{command} --traceback --settings=#{system}.envs.#{env} --pythonpath=. #{args.join(' ')}"
|
|
end
|
|
|
|
# Runs Process.spawn, and kills the process at the end of the rake process
|
|
# Expects the same arguments as Process.spawn
|
|
def background_process(*command)
|
|
pid = Process.spawn({}, *command, {:pgroup => true})
|
|
|
|
at_exit do
|
|
puts "Ending process and children"
|
|
pgid = Process.getpgid(pid)
|
|
begin
|
|
Timeout.timeout(5) do
|
|
puts "Terminating process group #{pgid}"
|
|
Process.kill(:SIGTERM, -pgid)
|
|
puts "Waiting on process group #{pgid}"
|
|
Process.wait(-pgid)
|
|
puts "Done waiting on process group #{pgid}"
|
|
end
|
|
rescue Timeout::Error
|
|
puts "Killing process group #{pgid}"
|
|
Process.kill(:SIGKILL, -pgid)
|
|
puts "Waiting on process group #{pgid}"
|
|
Process.wait(-pgid)
|
|
puts "Done waiting on process group #{pgid}"
|
|
end
|
|
end
|
|
end
|
|
|
|
def django_for_jasmine(system, django_reload)
|
|
if !django_reload
|
|
reload_arg = '--noreload'
|
|
end
|
|
|
|
port = 10000 + rand(40000)
|
|
jasmine_url = "http://localhost:#{port}/_jasmine/"
|
|
|
|
background_process(*django_admin(system, 'jasmine', 'runserver', '-v', '0', port.to_s, reload_arg).split(' '))
|
|
|
|
up = false
|
|
start_time = Time.now
|
|
until up do
|
|
if Time.now - start_time > 30
|
|
abort "Timed out waiting for server to start to run jasmine tests"
|
|
end
|
|
begin
|
|
response = Net::HTTP.get_response(URI(jasmine_url))
|
|
puts response.code
|
|
up = response.code == '200'
|
|
rescue => e
|
|
puts e.message
|
|
ensure
|
|
puts('Waiting server to start')
|
|
sleep(0.5)
|
|
end
|
|
end
|
|
yield jasmine_url
|
|
end
|
|
|
|
def template_jasmine_runner(lib)
|
|
coffee_files = Dir["#{lib}/**/js/**/*.coffee", "common/static/coffee/src/**/*.coffee"]
|
|
if !coffee_files.empty?
|
|
sh("coffee -c #{coffee_files.join(' ')}")
|
|
end
|
|
phantom_jasmine_path = File.expand_path("common/test/phantom-jasmine")
|
|
common_js_root = File.expand_path("common/static/js")
|
|
common_coffee_root = File.expand_path("common/static/coffee/src")
|
|
|
|
# Get arrays of spec and source files, ordered by how deep they are nested below the library
|
|
# (and then alphabetically) and expanded from a relative to an absolute path
|
|
spec_glob = File.join("#{lib}", "**", "spec", "**", "*.js")
|
|
src_glob = File.join("#{lib}", "**", "src", "**", "*.js")
|
|
js_specs = Dir[spec_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)}
|
|
js_source = Dir[src_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)}
|
|
|
|
template = ERB.new(File.read("#{lib}/jasmine_test_runner.html.erb"))
|
|
template_output = "#{lib}/jasmine_test_runner.html"
|
|
File.open(template_output, 'w') do |f|
|
|
f.write(template.result(binding))
|
|
end
|
|
yield File.expand_path(template_output)
|
|
end
|
|
|
|
|
|
def report_dir_path(dir)
|
|
return File.join(REPORT_DIR, dir.to_s)
|
|
end
|
|
|
|
def compile_assets(watch=false, debug=false)
|
|
xmodule_cmd = 'xmodule_assets common/static/xmodule'
|
|
if watch
|
|
xmodule_cmd = "watchmedo shell-command \
|
|
--patterns='*.js;*.coffee;*.sass;*.scss;*.css' \
|
|
--recursive \
|
|
--command='#{xmodule_cmd}' \
|
|
common/lib/xmodule"
|
|
end
|
|
coffee_cmd = "coffee #{watch ? '--watch' : ''} --compile */static"
|
|
sass_cmd = "sass #{debug ? '--debug-info' : '--style compressed'} " +
|
|
"--load-path ./common/static/sass " +
|
|
"--require ./common/static/sass/bourbon/lib/bourbon.rb " +
|
|
"#{watch ? '--watch' : '--update --force'} */static"
|
|
|
|
[xmodule_cmd, coffee_cmd, sass_cmd].each do |cmd|
|
|
if watch
|
|
background_process(cmd)
|
|
else
|
|
pid = Process.spawn(cmd)
|
|
puts "Waiting for `#{cmd}` to complete (pid #{pid})"
|
|
Process.wait(pid)
|
|
puts "Completed"
|
|
end
|
|
end
|
|
end
|
|
|
|
task :default => [:test, :pep8, :pylint]
|
|
|
|
directory REPORT_DIR
|
|
|
|
default_options = {
|
|
:lms => '8000',
|
|
:cms => '8001',
|
|
}
|
|
|
|
task :predjango do
|
|
sh("find . -type f -name *.pyc -delete")
|
|
sh('pip install -q --no-index -r local-requirements.txt')
|
|
end
|
|
|
|
task :clean_test_files do
|
|
sh("git clean -fqdx test_root")
|
|
end
|
|
|
|
[:lms, :cms, :common].each do |system|
|
|
report_dir = report_dir_path(system)
|
|
directory report_dir
|
|
|
|
desc "Run pep8 on all #{system} code"
|
|
task "pep8_#{system}" => report_dir do
|
|
sh("pep8 #{system} | tee #{report_dir}/pep8.report")
|
|
end
|
|
task :pep8 => "pep8_#{system}"
|
|
|
|
desc "Run pylint on all #{system} code"
|
|
task "pylint_#{system}" => report_dir do
|
|
Dir["#{system}/*.py", "#{system}/djangoapps/*", "#{system}/lib/*"].each do |app|
|
|
pythonpath_prefix = "PYTHONPATH=#{system}:#{system}/djangoapps:#{system}/lib:common/djangoapps:common/lib"
|
|
app = File.basename(app)
|
|
if app =~ /.py$/
|
|
app = app.gsub('.py', '')
|
|
elsif app =~ /.pyc$/
|
|
next
|
|
end
|
|
sh("#{pythonpath_prefix} pylint --rcfile=.pylintrc -f parseable #{app} | tee #{report_dir}/#{app}.pylint.report")
|
|
end
|
|
end
|
|
task :pylint => "pylint_#{system}"
|
|
|
|
end
|
|
|
|
$failed_tests = 0
|
|
|
|
def run_under_coverage(cmd, root)
|
|
cmd0, cmd_rest = cmd.split(" ", 2)
|
|
# We use "python -m coverage" so that the proper python will run the importable coverage
|
|
# rather than the coverage that OS path finds.
|
|
cmd = "python -m coverage run --rcfile=#{root}/.coveragerc `which #{cmd0}` #{cmd_rest}"
|
|
return cmd
|
|
end
|
|
|
|
def run_tests(system, report_dir, stop_on_failure=true)
|
|
ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
|
|
dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"]
|
|
cmd = django_admin(system, :test, 'test', '--logging-clear-handlers', *dirs.each)
|
|
sh(run_under_coverage(cmd, system)) do |ok, res|
|
|
if !ok and stop_on_failure
|
|
abort "Test failed!"
|
|
end
|
|
$failed_tests += 1 unless ok
|
|
end
|
|
end
|
|
|
|
TEST_TASK_DIRS = []
|
|
|
|
task :fastlms do
|
|
# this is >2 times faster that rake [lms], and does not need web, good for local dev
|
|
django_admin = ENV['DJANGO_ADMIN_PATH'] || select_executable('django-admin.py', 'django-admin')
|
|
sh("#{django_admin} runserver --traceback --settings=lms.envs.dev --pythonpath=.")
|
|
end
|
|
|
|
[:lms, :cms].each do |system|
|
|
report_dir = report_dir_path(system)
|
|
|
|
# Per System tasks
|
|
desc "Run all django tests on our djangoapps for the #{system}"
|
|
task "test_#{system}", [:stop_on_failure] => ["clean_test_files", "#{system}:gather_assets:test", "fasttest_#{system}"]
|
|
|
|
# Have a way to run the tests without running collectstatic -- useful when debugging without
|
|
# messing with static files.
|
|
task "fasttest_#{system}", [:stop_on_failure] => [report_dir, :predjango] do |t, args|
|
|
args.with_defaults(:stop_on_failure => 'true')
|
|
run_tests(system, report_dir, args.stop_on_failure)
|
|
end
|
|
|
|
task :fasttest => "fasttest_#{system}"
|
|
|
|
TEST_TASK_DIRS << system
|
|
|
|
desc <<-desc
|
|
Start the #{system} locally with the specified environment (defaults to dev).
|
|
Other useful environments are devplus (for dev testing with a real local database)
|
|
desc
|
|
task system, [:env, :options] => [:predjango] do |t, args|
|
|
args.with_defaults(:env => 'dev', :options => default_options[system])
|
|
|
|
# Compile all assets first
|
|
compile_assets(watch=false, debug=true)
|
|
|
|
# Listen for any changes to assets
|
|
compile_assets(watch=true, debug=true)
|
|
|
|
sh(django_admin(system, args.env, 'runserver', args.options))
|
|
end
|
|
|
|
# Per environment tasks
|
|
Dir["#{system}/envs/**/*.py"].each do |env_file|
|
|
env = env_file.gsub("#{system}/envs/", '').gsub(/\.py/, '').gsub('/', '.')
|
|
desc "Attempt to import the settings file #{system}.envs.#{env} and report any errors"
|
|
task "#{system}:check_settings:#{env}" => :predjango do
|
|
sh("echo 'import #{system}.envs.#{env}' | #{django_admin(system, env, 'shell')}")
|
|
end
|
|
|
|
desc "Compile coffeescript and sass, and then run collectstatic in the specified environment"
|
|
task "#{system}:gather_assets:#{env}" do
|
|
compile_assets()
|
|
sh("#{django_admin(system, env, 'collectstatic', '--noinput')} > /dev/null") do |ok, status|
|
|
if !ok
|
|
abort "collectstatic failed!"
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
desc "Open jasmine tests for #{system} in your default browser"
|
|
task "browse_jasmine_#{system}" do
|
|
django_for_jasmine(system, true) do |jasmine_url|
|
|
Launchy.open(jasmine_url)
|
|
puts "Press ENTER to terminate".red
|
|
$stdin.gets
|
|
end
|
|
end
|
|
|
|
desc "Use phantomjs to run jasmine tests for #{system} from the console"
|
|
task "phantomjs_jasmine_#{system}" do
|
|
phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs'
|
|
django_for_jasmine(system, false) do |jasmine_url|
|
|
sh("#{phantomjs} common/test/phantom-jasmine/lib/run_jasmine_test.coffee #{jasmine_url}")
|
|
end
|
|
end
|
|
end
|
|
|
|
desc "Reset the relational database used by django. WARNING: this will delete all of your existing users"
|
|
task :resetdb, [:env] do |t, args|
|
|
args.with_defaults(:env => 'dev')
|
|
sh(django_admin(:lms, args.env, 'syncdb'))
|
|
sh(django_admin(:lms, args.env, 'migrate'))
|
|
end
|
|
|
|
desc "Update the relational database to the latest migration"
|
|
task :migrate, [:env] do |t, args|
|
|
args.with_defaults(:env => 'dev')
|
|
sh(django_admin(:lms, args.env, 'migrate'))
|
|
end
|
|
|
|
Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib|
|
|
task_name = "test_#{lib}"
|
|
|
|
report_dir = report_dir_path(lib)
|
|
|
|
desc "Run tests for common lib #{lib}"
|
|
task task_name => report_dir do
|
|
ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
|
|
cmd = "nosetests #{lib}"
|
|
sh(run_under_coverage(cmd, lib)) do |ok, res|
|
|
$failed_tests += 1 unless ok
|
|
end
|
|
end
|
|
TEST_TASK_DIRS << lib
|
|
|
|
desc "Run tests for common lib #{lib} (without coverage)"
|
|
task "fasttest_#{lib}" do
|
|
sh("nosetests #{lib}")
|
|
end
|
|
|
|
desc "Open jasmine tests for #{lib} in your default browser"
|
|
task "browse_jasmine_#{lib}" do
|
|
template_jasmine_runner(lib) do |f|
|
|
sh("python -m webbrowser -t 'file://#{f}'")
|
|
puts "Press ENTER to terminate".red
|
|
$stdin.gets
|
|
end
|
|
end
|
|
|
|
desc "Use phantomjs to run jasmine tests for #{lib} from the console"
|
|
task "phantomjs_jasmine_#{lib}" do
|
|
phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs'
|
|
template_jasmine_runner(lib) do |f|
|
|
sh("#{phantomjs} common/test/phantom-jasmine/lib/run_jasmine_test.coffee #{f}")
|
|
end
|
|
end
|
|
end
|
|
|
|
task :report_dirs
|
|
|
|
TEST_TASK_DIRS.each do |dir|
|
|
report_dir = report_dir_path(dir)
|
|
directory report_dir
|
|
task :report_dirs => [REPORT_DIR, report_dir]
|
|
end
|
|
|
|
task :test do
|
|
TEST_TASK_DIRS.each do |dir|
|
|
Rake::Task["test_#{dir}"].invoke(false)
|
|
end
|
|
|
|
if $failed_tests > 0
|
|
abort "Tests failed!"
|
|
end
|
|
end
|
|
|
|
namespace :coverage do
|
|
desc "Build the html coverage reports"
|
|
task :html => :report_dirs do
|
|
TEST_TASK_DIRS.each do |dir|
|
|
report_dir = report_dir_path(dir)
|
|
|
|
if !File.file?("#{report_dir}/.coverage")
|
|
next
|
|
end
|
|
|
|
sh("coverage html --rcfile=#{dir}/.coveragerc")
|
|
end
|
|
end
|
|
|
|
desc "Build the xml coverage reports"
|
|
task :xml => :report_dirs do
|
|
TEST_TASK_DIRS.each do |dir|
|
|
report_dir = report_dir_path(dir)
|
|
|
|
if !File.file?("#{report_dir}/.coverage")
|
|
next
|
|
end
|
|
# Why doesn't the rcfile control the xml output file properly??
|
|
sh("coverage xml -o #{report_dir}/coverage.xml --rcfile=#{dir}/.coveragerc")
|
|
end
|
|
end
|
|
end
|
|
|
|
task :runserver => :lms
|
|
|
|
desc "Run django-admin <action> against the specified system and environment"
|
|
task "django-admin", [:action, :system, :env, :options] do |t, args|
|
|
args.with_defaults(:env => 'dev', :system => 'lms', :options => '')
|
|
sh(django_admin(args.system, args.env, args.action, args.options))
|
|
end
|
|
|
|
desc "Set the staff bit for a user"
|
|
task :set_staff, [:user, :system, :env] do |t, args|
|
|
args.with_defaults(:env => 'dev', :system => 'lms', :options => '')
|
|
sh(django_admin(args.system, args.env, 'set_staff', args.user))
|
|
end
|
|
|
|
namespace :cms do
|
|
desc "Clone existing MongoDB based course"
|
|
task :clone do
|
|
|
|
if ENV['SOURCE_LOC'] and ENV['DEST_LOC']
|
|
sh(django_admin(:cms, :dev, :clone, ENV['SOURCE_LOC'], ENV['DEST_LOC']))
|
|
else
|
|
raise "You must pass in a SOURCE_LOC and DEST_LOC parameters"
|
|
end
|
|
end
|
|
|
|
desc "Delete existing MongoDB based course"
|
|
task :delete_course do
|
|
|
|
if ENV['LOC'] and ENV['COMMIT']
|
|
sh(django_admin(:cms, :dev, :delete_course, ENV['LOC'], ENV['COMMIT']))
|
|
elsif ENV['LOC']
|
|
sh(django_admin(:cms, :dev, :delete_course, ENV['LOC']))
|
|
else
|
|
raise "You must pass in a LOC parameter"
|
|
end
|
|
end
|
|
|
|
desc "Import course data within the given DATA_DIR variable"
|
|
task :import do
|
|
if ENV['DATA_DIR'] and ENV['COURSE_DIR']
|
|
sh(django_admin(:cms, :dev, :import, ENV['DATA_DIR'], ENV['COURSE_DIR']))
|
|
elsif ENV['DATA_DIR']
|
|
sh(django_admin(:cms, :dev, :import, ENV['DATA_DIR']))
|
|
else
|
|
raise "Please specify a DATA_DIR variable that point to your data directory.\n" +
|
|
"Example: \`rake cms:import DATA_DIR=../data\`"
|
|
end
|
|
end
|
|
|
|
desc "Imports all the templates from the code pack"
|
|
task :update_templates do
|
|
sh(django_admin(:cms, :dev, :update_templates))
|
|
end
|
|
|
|
desc "Import course data within the given DATA_DIR variable"
|
|
task :xlint do
|
|
if ENV['DATA_DIR'] and ENV['COURSE_DIR']
|
|
sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR'], ENV['COURSE_DIR']))
|
|
elsif ENV['DATA_DIR']
|
|
sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR']))
|
|
else
|
|
raise "Please specify a DATA_DIR variable that point to your data directory.\n" +
|
|
"Example: \`rake cms:import DATA_DIR=../data\`"
|
|
end
|
|
end
|
|
|
|
desc "Export course data to a tar.gz file"
|
|
task :export do
|
|
if ENV['COURSE_ID'] and ENV['OUTPUT_PATH']
|
|
sh(django_admin(:cms, :dev, :export, ENV['COURSE_ID'], ENV['OUTPUT_PATH']))
|
|
else
|
|
raise "Please specify a COURSE_ID and OUTPUT_PATH.\n" +
|
|
"Example: \`rake cms:export COURSE_ID=MITx/12345/name OUTPUT_PATH=foo.tar.gz\`"
|
|
end
|
|
end
|
|
end
|
|
|
|
desc "Build a properties file used to trigger autodeploy builds"
|
|
task :autodeploy_properties do
|
|
File.open("autodeploy.properties", "w") do |file|
|
|
file.puts("UPSTREAM_NOOP=false")
|
|
file.puts("UPSTREAM_BRANCH=#{BRANCH}")
|
|
file.puts("UPSTREAM_JOB=#{PACKAGE_NAME}")
|
|
file.puts("UPSTREAM_REVISION=#{COMMIT}")
|
|
end
|
|
end
|
|
|
|
|
|
# --- Develop and public documentation ---
|
|
desc "Invoke sphinx 'make build' to generate docs."
|
|
task :builddocs, [:options] do |t, args|
|
|
if args.options == 'pub'
|
|
path = "doc/public"
|
|
else
|
|
path = "docs"
|
|
end
|
|
|
|
Dir.chdir(path) do
|
|
sh('make html')
|
|
end
|
|
end
|
|
|
|
desc "Show docs in browser (mac and ubuntu)."
|
|
task :showdocs, [:options] do |t, args|
|
|
if args.options == 'pub'
|
|
path = "doc/public"
|
|
else
|
|
path = "docs"
|
|
end
|
|
|
|
Dir.chdir("#{path}/build/html") do
|
|
if RUBY_PLATFORM.include? 'darwin' # mac os
|
|
sh('open index.html')
|
|
elsif RUBY_PLATFORM.include? 'linux' # make more ubuntu specific?
|
|
sh('sensible-browser index.html') # ubuntu
|
|
else
|
|
raise "\nUndefined how to run browser on your machine.
|
|
Please use 'rake builddocs' and then manually open
|
|
'mitx/#{path}/build/html/index.html."
|
|
end
|
|
end
|
|
end
|
|
|
|
desc "Build docs and show them in browser"
|
|
task :doc, [:options] => :builddocs do |t, args|
|
|
Rake::Task["showdocs"].invoke(args.options)
|
|
end
|
|
# --- Develop and public documentation ---
|