diff --git a/rakefile b/rakefile index f7c5b71306..cef93e67eb 100644 --- a/rakefile +++ b/rakefile @@ -1,670 +1,12 @@ require 'rake/clean' -require 'tempfile' -require 'net/http' -require 'launchy' -require 'colorize' -require 'erb' -require 'tempfile' -require 'digest/md5' +require './rakefiles/helpers.rb' + +Dir['rakefiles/*.rake'].each do |rakefile| + import rakefile +end # Build Constants REPO_ROOT = File.dirname(__FILE__) -BUILD_DIR = File.join(REPO_ROOT, "build") REPORT_DIR = File.join(REPO_ROOT, "reports") -# Packaging constants -PACKAGE_NAME = "edx-platform" -COMMIT = (ENV["GIT_COMMIT"] || `git rev-parse HEAD`).chomp()[0, 10] -BRANCH = (ENV["GIT_BRANCH"] || `git symbolic-ref -q HEAD`).chomp().gsub('refs/heads/', '').gsub('origin/', '') - -PREREQS_MD5_DIR = ENV["PREREQ_CACHE_DIR"] || File.join(REPO_ROOT, '.prereqs_cache') - -# Set up the clean and clobber tasks -CLOBBER.include(BUILD_DIR, REPORT_DIR, 'test_root/*_repo', 'test_root/staticfiles', PREREQS_MD5_DIR) -CLEAN.include("#{BUILD_DIR}/*.deb", "#{BUILD_DIR}/util") - -def select_executable(*cmds) - cmds.find_all{ |cmd| system("which #{cmd} > /dev/null 2>&1") }[0] || fail("No executables found from #{cmds.join(', ')}") -end - -def django_admin(system, env, command, *args) - django_admin = ENV['DJANGO_ADMIN_PATH'] || select_executable('django-admin.py', 'django-admin') - return "#{django_admin} #{command} --traceback --settings=#{system}.envs.#{env} --pythonpath=. #{args.join(' ')}" -end - -# Runs Process.spawn, and kills the process at the end of the rake process -# Expects the same arguments as Process.spawn -def background_process(*command) - pid = Process.spawn({}, *command, {:pgroup => true}) - - at_exit do - puts "Ending process and children" - pgid = Process.getpgid(pid) - begin - Timeout.timeout(5) do - puts "Terminating process group #{pgid}" - Process.kill(:SIGTERM, -pgid) - puts "Waiting on process group #{pgid}" - Process.wait(-pgid) - puts "Done waiting on process group #{pgid}" - end - rescue Timeout::Error - puts "Killing process group #{pgid}" - Process.kill(:SIGKILL, -pgid) - puts "Waiting on process group #{pgid}" - Process.wait(-pgid) - puts "Done waiting on process group #{pgid}" - end - end -end - -def django_for_jasmine(system, django_reload) - if !django_reload - reload_arg = '--noreload' - end - - port = 10000 + rand(40000) - jasmine_url = "http://localhost:#{port}/_jasmine/" - - background_process(*django_admin(system, 'jasmine', 'runserver', '-v', '0', port.to_s, reload_arg).split(' ')) - - up = false - start_time = Time.now - until up do - if Time.now - start_time > 30 - abort "Timed out waiting for server to start to run jasmine tests" - end - begin - response = Net::HTTP.get_response(URI(jasmine_url)) - puts response.code - up = response.code == '200' - rescue => e - puts e.message - ensure - puts('Waiting server to start') - sleep(0.5) - end - end - yield jasmine_url -end - -def template_jasmine_runner(lib) - coffee_files = Dir["#{lib}/**/js/**/*.coffee", "common/static/coffee/src/**/*.coffee"] - if !coffee_files.empty? - sh("node_modules/.bin/coffee -c #{coffee_files.join(' ')}") - end - phantom_jasmine_path = File.expand_path("node_modules/phantom-jasmine") - common_js_root = File.expand_path("common/static/js") - common_coffee_root = File.expand_path("common/static/coffee/src") - - # Get arrays of spec and source files, ordered by how deep they are nested below the library - # (and then alphabetically) and expanded from a relative to an absolute path - spec_glob = File.join("#{lib}", "**", "spec", "**", "*.js") - src_glob = File.join("#{lib}", "**", "src", "**", "*.js") - js_specs = Dir[spec_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)} - js_source = Dir[src_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)} - - template = ERB.new(File.read("#{lib}/jasmine_test_runner.html.erb")) - template_output = "#{lib}/jasmine_test_runner.html" - File.open(template_output, 'w') do |f| - f.write(template.result(binding)) - end - yield File.expand_path(template_output) -end - - -def report_dir_path(dir) - return File.join(REPORT_DIR, dir.to_s) -end - -def xmodule_cmd(watch=false, debug=false) - xmodule_cmd = 'xmodule_assets common/static/xmodule' - if watch - "watchmedo shell-command " + - "--patterns='*.js;*.coffee;*.sass;*.scss;*.css' " + - "--recursive " + - "--command='#{xmodule_cmd}' " + - "common/lib/xmodule" - else - xmodule_cmd - end -end - -def coffee_cmd(watch=false, debug=false) - "node_modules/.bin/coffee #{watch ? '--watch' : ''} --compile */static" -end - -def sass_cmd(watch=false, debug=false) - "sass #{debug ? '--debug-info' : '--style compressed'} " + - "--load-path ./common/static/sass " + - "--require ./common/static/sass/bourbon/lib/bourbon.rb " + - "#{watch ? '--watch' : '--update'} */static" -end - -desc "Compile all assets" -multitask :assets => 'assets:all' - -desc "Compile all assets in debug mode" -multitask 'assets:debug' - -desc "Watch all assets for changes and automatically recompile" -multitask 'assets:watch' - -namespace :assets do - {:xmodule => :install_python_prereqs, - :coffee => :install_node_prereqs, - :sass => :install_ruby_prereqs}.each_pair do |asset_type, prereq_task| - desc "Compile all #{asset_type} assets" - task asset_type => prereq_task do - cmd = send(asset_type.to_s + "_cmd", watch=false, debug=false) - sh(cmd) - end - - multitask :all => asset_type - multitask :debug => "assets:#{asset_type}:debug" - multitask :watch => "assets:#{asset_type}:watch" - - namespace asset_type do - desc "Compile all #{asset_type} assets in debug mode" - task :debug => prereq_task do - cmd = send(asset_type.to_s + "_cmd", watch=false, debug=true) - sh(cmd) - end - - desc "Watch all #{asset_type} assets and compile on change" - task :watch => prereq_task do - cmd = send(asset_type.to_s + "_cmd", watch=true, debug=true) - background_process(cmd) - end - end - end - - # In watch mode, sass doesn't immediately compile out of date files, - # so force a recompile first - task "sass:watch" => "assets:sass:debug" - - multitask :sass => 'assets:xmodule' - multitask 'sass:debug' => 'assets:xmodule:debug' - multitask :coffee => 'assets:xmodule' - multitask 'coffee:debug' => 'assets:xmodule:debug' -end - - -directory PREREQS_MD5_DIR - -def when_changed(*files) - Rake::Task[PREREQS_MD5_DIR].invoke - cache_file = File.join(PREREQS_MD5_DIR, files.join('-').gsub(/\W+/, '-')) + '.md5' - digest = Digest::MD5.new() - Dir[*files].select{|file| File.file?(file)}.each do |file| - digest.file(file) - end - if !File.exists?(cache_file) or digest.hexdigest != File.read(cache_file) - yield - File.write(cache_file, digest.hexdigest) - end -end - task :default => [:test, :pep8, :pylint] - -directory REPORT_DIR - -default_options = { - :lms => '8000', - :cms => '8001', -} - -desc "Install all prerequisites needed for the lms and cms" -task :install_prereqs => [:install_node_prereqs, :install_ruby_prereqs, :install_python_prereqs] - -desc "Install all node prerequisites for the lms and cms" -task :install_node_prereqs do - when_changed('package.json') do - sh('npm install') - end unless ENV['NO_PREREQ_INSTALL'] -end - -desc "Install all ruby prerequisites for the lms and cms" -task :install_ruby_prereqs do - when_changed('Gemfile') do - sh('bundle install') - end unless ENV['NO_PREREQ_INSTALL'] -end - -desc "Install all python prerequisites for the lms and cms" -task :install_python_prereqs do - when_changed('requirements/**') do - ENV['PIP_DOWNLOAD_CACHE'] ||= '.pip_download_cache' - sh('pip install --exists-action w -r requirements/base.txt') - sh('pip install --exists-action w -r requirements/post.txt') - # Check for private-requirements.txt: used to install our libs as working dirs, - # or personal-use tools. - if File.file?("requirements/private.txt") - sh('pip install -r requirements/private.txt') - end - end unless ENV['NO_PREREQ_INSTALL'] -end - -task :predjango do - sh("find . -type f -name *.pyc -delete") - sh('pip install -q --no-index -r requirements/local.txt') -end - -task :clean_test_files do - sh("git clean -fqdx test_root") -end - -[:lms, :cms, :common].each do |system| - report_dir = report_dir_path(system) - directory report_dir - - desc "Run pep8 on all #{system} code" - task "pep8_#{system}" => report_dir do - sh("pep8 #{system} | tee #{report_dir}/pep8.report") - end - task :pep8 => "pep8_#{system}" - - desc "Run pylint on all #{system} code" - task "pylint_#{system}" => report_dir do - apps = Dir["#{system}/*.py", "#{system}/djangoapps/*", "#{system}/lib/*"].map do |app| - File.basename(app) - end.select do |app| - app !=~ /.pyc$/ - end.map do |app| - if app =~ /.py$/ - app.gsub('.py', '') - else - app - end - end - - pythonpath_prefix = "PYTHONPATH=#{system}:#{system}/djangoapps:#{system}/lib:common/djangoapps:common/lib" - sh("#{pythonpath_prefix} pylint --rcfile=.pylintrc -f parseable #{apps.join(' ')} | tee #{report_dir}/pylint.report") - end - task :pylint => "pylint_#{system}" - -end - -$failed_tests = 0 - -def run_under_coverage(cmd, root) - cmd0, cmd_rest = cmd.split(" ", 2) - # We use "python -m coverage" so that the proper python will run the importable coverage - # rather than the coverage that OS path finds. - cmd = "python -m coverage run --rcfile=#{root}/.coveragerc `which #{cmd0}` #{cmd_rest}" - return cmd -end - -def run_tests(system, report_dir, stop_on_failure=true) - ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml") - dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"] - cmd = django_admin(system, :test, 'test', '--logging-clear-handlers', *dirs.each) - sh(run_under_coverage(cmd, system)) do |ok, res| - if !ok and stop_on_failure - abort "Test failed!" - end - $failed_tests += 1 unless ok - end -end - -TEST_TASK_DIRS = [] - -task :fastlms do - # this is >2 times faster that rake [lms], and does not need web, good for local dev - django_admin = ENV['DJANGO_ADMIN_PATH'] || select_executable('django-admin.py', 'django-admin') - sh("#{django_admin} runserver --traceback --settings=lms.envs.dev --pythonpath=.") -end - -[:lms, :cms].each do |system| - report_dir = report_dir_path(system) - - # Per System tasks - desc "Run all django tests on our djangoapps for the #{system}" - task "test_#{system}", [:stop_on_failure] => ["clean_test_files", :predjango, "#{system}:gather_assets:test", "fasttest_#{system}"] - - # Have a way to run the tests without running collectstatic -- useful when debugging without - # messing with static files. - task "fasttest_#{system}", [:stop_on_failure] => [report_dir, :install_prereqs, :predjango] do |t, args| - args.with_defaults(:stop_on_failure => 'true') - run_tests(system, report_dir, args.stop_on_failure) - end - - task :fasttest => "fasttest_#{system}" - - TEST_TASK_DIRS << system - - desc <<-desc - Start the #{system} locally with the specified environment (defaults to dev). - Other useful environments are devplus (for dev testing with a real local database) - desc - task system, [:env, :options] => [:install_prereqs, 'assets:watch', :predjango] do |t, args| - args.with_defaults(:env => 'dev', :options => default_options[system]) - sh(django_admin(system, args.env, 'runserver', args.options)) - end - - # Per environment tasks - Dir["#{system}/envs/**/*.py"].each do |env_file| - env = env_file.gsub("#{system}/envs/", '').gsub(/\.py/, '').gsub('/', '.') - desc "Attempt to import the settings file #{system}.envs.#{env} and report any errors" - task "#{system}:check_settings:#{env}" => :predjango do - sh("echo 'import #{system}.envs.#{env}' | #{django_admin(system, env, 'shell')}") - end - - desc "Compile coffeescript and sass, and then run collectstatic in the specified environment" - task "#{system}:gather_assets:#{env}" => :assets do - sh("#{django_admin(system, env, 'collectstatic', '--noinput')} > /dev/null") do |ok, status| - if !ok - abort "collectstatic failed!" - end - end - end - end - - desc "Open jasmine tests for #{system} in your default browser" - task "browse_jasmine_#{system}" => :assets do - django_for_jasmine(system, true) do |jasmine_url| - Launchy.open(jasmine_url) - puts "Press ENTER to terminate".red - $stdin.gets - end - end - - desc "Use phantomjs to run jasmine tests for #{system} from the console" - task "phantomjs_jasmine_#{system}" => :assets do - phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs' - django_for_jasmine(system, false) do |jasmine_url| - sh("#{phantomjs} node_modules/phantom-jasmine/lib/run_jasmine_test.coffee #{jasmine_url}") - end - end -end - -desc "Reset the relational database used by django. WARNING: this will delete all of your existing users" -task :resetdb, [:env] do |t, args| - args.with_defaults(:env => 'dev') - sh(django_admin(:lms, args.env, 'syncdb')) - sh(django_admin(:lms, args.env, 'migrate')) -end - -desc "Update the relational database to the latest migration" -task :migrate, [:env] do |t, args| - args.with_defaults(:env => 'dev') - sh(django_admin(:lms, args.env, 'migrate')) -end - -Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib| - task_name = "test_#{lib}" - - report_dir = report_dir_path(lib) - - desc "Run tests for common lib #{lib}" - task task_name => report_dir do - ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml") - cmd = "nosetests #{lib}" - sh(run_under_coverage(cmd, lib)) do |ok, res| - $failed_tests += 1 unless ok - end - end - TEST_TASK_DIRS << lib - - desc "Run tests for common lib #{lib} (without coverage)" - task "fasttest_#{lib}" do - sh("nosetests #{lib}") - end - - desc "Open jasmine tests for #{lib} in your default browser" - task "browse_jasmine_#{lib}" do - template_jasmine_runner(lib) do |f| - sh("python -m webbrowser -t 'file://#{f}'") - puts "Press ENTER to terminate".red - $stdin.gets - end - end - - desc "Use phantomjs to run jasmine tests for #{lib} from the console" - task "phantomjs_jasmine_#{lib}" do - phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs' - template_jasmine_runner(lib) do |f| - sh("#{phantomjs} node_modules/phantom-jasmine/lib/run_jasmine_test.coffee #{f}") - end - end -end - -task :report_dirs - -TEST_TASK_DIRS.each do |dir| - report_dir = report_dir_path(dir) - directory report_dir - task :report_dirs => [REPORT_DIR, report_dir] -end - -task :test do - TEST_TASK_DIRS.each do |dir| - Rake::Task["test_#{dir}"].invoke(false) - end - - if $failed_tests > 0 - abort "Tests failed!" - end -end - -namespace :coverage do - desc "Build the html coverage reports" - task :html => :report_dirs do - TEST_TASK_DIRS.each do |dir| - report_dir = report_dir_path(dir) - - if !File.file?("#{report_dir}/.coverage") - next - end - - sh("coverage html --rcfile=#{dir}/.coveragerc") - end - end - - desc "Build the xml coverage reports" - task :xml => :report_dirs do - TEST_TASK_DIRS.each do |dir| - report_dir = report_dir_path(dir) - - if !File.file?("#{report_dir}/.coverage") - next - end - # Why doesn't the rcfile control the xml output file properly?? - sh("coverage xml -o #{report_dir}/coverage.xml --rcfile=#{dir}/.coveragerc") - end - end -end - -task :runserver => :lms - -desc "Run django-admin against the specified system and environment" -task "django-admin", [:action, :system, :env, :options] do |t, args| - args.with_defaults(:env => 'dev', :system => 'lms', :options => '') - sh(django_admin(args.system, args.env, args.action, args.options)) -end - -desc "Set the staff bit for a user" -task :set_staff, [:user, :system, :env] do |t, args| - args.with_defaults(:env => 'dev', :system => 'lms', :options => '') - sh(django_admin(args.system, args.env, 'set_staff', args.user)) -end - -namespace :cms do - desc "Clone existing MongoDB based course" - task :clone do - - if ENV['SOURCE_LOC'] and ENV['DEST_LOC'] - sh(django_admin(:cms, :dev, :clone, ENV['SOURCE_LOC'], ENV['DEST_LOC'])) - else - raise "You must pass in a SOURCE_LOC and DEST_LOC parameters" - end - end - - desc "Delete existing MongoDB based course" - task :delete_course do - - if ENV['LOC'] and ENV['COMMIT'] - sh(django_admin(:cms, :dev, :delete_course, ENV['LOC'], ENV['COMMIT'])) - elsif ENV['LOC'] - sh(django_admin(:cms, :dev, :delete_course, ENV['LOC'])) - else - raise "You must pass in a LOC parameter" - end - end - - desc "Import course data within the given DATA_DIR variable" - task :import do - if ENV['DATA_DIR'] and ENV['COURSE_DIR'] - sh(django_admin(:cms, :dev, :import, ENV['DATA_DIR'], ENV['COURSE_DIR'])) - elsif ENV['DATA_DIR'] - sh(django_admin(:cms, :dev, :import, ENV['DATA_DIR'])) - else - raise "Please specify a DATA_DIR variable that point to your data directory.\n" + - "Example: \`rake cms:import DATA_DIR=../data\`" - end - end - - desc "Imports all the templates from the code pack" - task :update_templates do - sh(django_admin(:cms, :dev, :update_templates)) - end - - desc "Import course data within the given DATA_DIR variable" - task :xlint do - if ENV['DATA_DIR'] and ENV['COURSE_DIR'] - sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR'], ENV['COURSE_DIR'])) - elsif ENV['DATA_DIR'] - sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR'])) - else - raise "Please specify a DATA_DIR variable that point to your data directory.\n" + - "Example: \`rake cms:import DATA_DIR=../data\`" - end - end - - desc "Export course data to a tar.gz file" - task :export do - if ENV['COURSE_ID'] and ENV['OUTPUT_PATH'] - sh(django_admin(:cms, :dev, :export, ENV['COURSE_ID'], ENV['OUTPUT_PATH'])) - else - raise "Please specify a COURSE_ID and OUTPUT_PATH.\n" + - "Example: \`rake cms:export COURSE_ID=MITx/12345/name OUTPUT_PATH=foo.tar.gz\`" - end - end -end - -desc "Build a properties file used to trigger autodeploy builds" -task :autodeploy_properties do - File.open("autodeploy.properties", "w") do |file| - file.puts("UPSTREAM_NOOP=false") - file.puts("UPSTREAM_BRANCH=#{BRANCH}") - file.puts("UPSTREAM_JOB=#{PACKAGE_NAME}") - file.puts("UPSTREAM_REVISION=#{COMMIT}") - end -end - -# --- Internationalization tasks - -namespace :i18n do - - desc "Extract localizable strings from sources" - task :extract => "i18n:validate:gettext" do - sh(File.join(REPO_ROOT, "i18n", "extract.py")) - end - - desc "Compile localizable strings from sources. With optional flag 'extract', will extract strings first." - task :generate => "i18n:validate:gettext" do - if ARGV.last.downcase == 'extract' - Rake::Task["i18n:extract"].execute - end - sh(File.join(REPO_ROOT, "i18n", "generate.py")) - end - - desc "Simulate international translation by generating dummy strings corresponding to source strings." - task :dummy do - source_files = Dir["#{REPO_ROOT}/conf/locale/en/LC_MESSAGES/*.po"] - dummy_locale = 'fr' - cmd = File.join(REPO_ROOT, "i18n", "make_dummy.py") - for file in source_files do - sh("#{cmd} #{file} #{dummy_locale}") - end - end - - namespace :validate do - - desc "Make sure GNU gettext utilities are available" - task :gettext do - begin - select_executable('xgettext') - rescue - msg = "Cannot locate GNU gettext utilities, which are required by django for internationalization.\n" - msg += "(see https://docs.djangoproject.com/en/dev/topics/i18n/translation/#message-files)\n" - msg += "Try downloading them from http://www.gnu.org/software/gettext/" - abort(msg.red) - end - end - - desc "Make sure config file with username/password exists" - task :transifex_config do - config_file = "#{Dir.home}/.transifexrc" - if !File.file?(config_file) or File.size(config_file)==0 - msg ="Cannot connect to Transifex, config file is missing or empty: #{config_file}\n" - msg += "See http://help.transifex.com/features/client/#transifexrc" - abort(msg.red) - end - end - end - - namespace :transifex do - desc "Push source strings to Transifex for translation" - task :push => "i18n:validate:transifex_config" do - cmd = File.join(REPO_ROOT, "i18n", "transifex.py") - sh("#{cmd} push") - end - - desc "Pull translated strings from Transifex" - task :pull => "i18n:validate:transifex_config" do - cmd = File.join(REPO_ROOT, "i18n", "transifex.py") - sh("#{cmd} pull") - end - end - - desc "Run tests for the internationalization library" - task :test => "i18n:validate:gettext" do - test = File.join(REPO_ROOT, "i18n", "tests") - sh("nosetests #{test}") - end - -end - -# --- Develop and public documentation --- -desc "Invoke sphinx 'make build' to generate docs." -task :builddocs, [:options] do |t, args| - if args.options == 'pub' - path = "doc/public" - else - path = "docs" - end - - Dir.chdir(path) do - sh('make html') - end -end - -desc "Show docs in browser (mac and ubuntu)." -task :showdocs, [:options] do |t, args| - if args.options == 'pub' - path = "doc/public" - else - path = "docs" - end - - Dir.chdir("#{path}/build/html") do - Launchy.open('index.html') - end -end - -desc "Build docs and show them in browser" -task :doc, [:options] => :builddocs do |t, args| - Rake::Task["showdocs"].invoke(args.options) -end -# --- Develop and public documentation --- diff --git a/rakefiles/assets.rake b/rakefiles/assets.rake new file mode 100644 index 0000000000..a11378ae13 --- /dev/null +++ b/rakefiles/assets.rake @@ -0,0 +1,86 @@ + +def xmodule_cmd(watch=false, debug=false) + xmodule_cmd = 'xmodule_assets common/static/xmodule' + if watch + "watchmedo shell-command " + + "--patterns='*.js;*.coffee;*.sass;*.scss;*.css' " + + "--recursive " + + "--command='#{xmodule_cmd}' " + + "common/lib/xmodule" + else + xmodule_cmd + end +end + +def coffee_cmd(watch=false, debug=false) + "node_modules/.bin/coffee #{watch ? '--watch' : ''} --compile */static" +end + +def sass_cmd(watch=false, debug=false) + "sass #{debug ? '--debug-info' : '--style compressed'} " + + "--load-path ./common/static/sass " + + "--require ./common/static/sass/bourbon/lib/bourbon.rb " + + "#{watch ? '--watch' : '--update'} */static" +end + +desc "Compile all assets" +multitask :assets => 'assets:all' + +desc "Compile all assets in debug mode" +multitask 'assets:debug' + +desc "Watch all assets for changes and automatically recompile" +multitask 'assets:watch' + +namespace :assets do + {:xmodule => :install_python_prereqs, + :coffee => :install_node_prereqs, + :sass => :install_ruby_prereqs}.each_pair do |asset_type, prereq_task| + desc "Compile all #{asset_type} assets" + task asset_type => prereq_task do + cmd = send(asset_type.to_s + "_cmd", watch=false, debug=false) + sh(cmd) + end + + multitask :all => asset_type + multitask :debug => "assets:#{asset_type}:debug" + multitask :watch => "assets:#{asset_type}:watch" + + namespace asset_type do + desc "Compile all #{asset_type} assets in debug mode" + task :debug => prereq_task do + cmd = send(asset_type.to_s + "_cmd", watch=false, debug=true) + sh(cmd) + end + + desc "Watch all #{asset_type} assets and compile on change" + task :watch => prereq_task do + cmd = send(asset_type.to_s + "_cmd", watch=true, debug=true) + background_process(cmd) + end + end + end + + # In watch mode, sass doesn't immediately compile out of date files, + # so force a recompile first + task "sass:watch" => "assets:sass:debug" + + multitask :sass => 'assets:xmodule' + multitask 'sass:debug' => 'assets:xmodule:debug' + multitask :coffee => 'assets:xmodule' + multitask 'coffee:debug' => 'assets:xmodule:debug' +end + +[:lms, :cms].each do |system| + # Per environment tasks + environments(system).each do |env| + desc "Compile coffeescript and sass, and then run collectstatic in the specified environment" + task "#{system}:gather_assets:#{env}" => :assets do + sh("#{django_admin(system, env, 'collectstatic', '--noinput')} > /dev/null") do |ok, status| + if !ok + abort "collectstatic failed!" + end + end + end + end +end diff --git a/rakefiles/deploy.rake b/rakefiles/deploy.rake new file mode 100644 index 0000000000..1d0a1b2c4f --- /dev/null +++ b/rakefiles/deploy.rake @@ -0,0 +1,15 @@ + +# Packaging constants +COMMIT = (ENV["GIT_COMMIT"] || `git rev-parse HEAD`).chomp()[0, 10] +PACKAGE_NAME = "mitx" +BRANCH = (ENV["GIT_BRANCH"] || `git symbolic-ref -q HEAD`).chomp().gsub('refs/heads/', '').gsub('origin/', '') + +desc "Build a properties file used to trigger autodeploy builds" +task :autodeploy_properties do + File.open("autodeploy.properties", "w") do |file| + file.puts("UPSTREAM_NOOP=false") + file.puts("UPSTREAM_BRANCH=#{BRANCH}") + file.puts("UPSTREAM_JOB=#{PACKAGE_NAME}") + file.puts("UPSTREAM_REVISION=#{COMMIT}") + end +end \ No newline at end of file diff --git a/rakefiles/django.rake b/rakefiles/django.rake new file mode 100644 index 0000000000..5f66c7d0c1 --- /dev/null +++ b/rakefiles/django.rake @@ -0,0 +1,125 @@ +default_options = { + :lms => '8000', + :cms => '8001', +} + +task :predjango do + sh("find . -type f -name *.pyc -delete") + sh('pip install -q --no-index -r requirements/local.txt') +end + + +task :fastlms do + # this is >2 times faster that rake [lms], and does not need web, good for local dev + django_admin = ENV['DJANGO_ADMIN_PATH'] || select_executable('django-admin.py', 'django-admin') + sh("#{django_admin} runserver --traceback --settings=lms.envs.dev --pythonpath=.") +end + +[:lms, :cms].each do |system| + desc <<-desc + Start the #{system} locally with the specified environment (defaults to dev). + Other useful environments are devplus (for dev testing with a real local database) + desc + task system, [:env, :options] => [:install_prereqs, 'assets:watch', :predjango] do |t, args| + args.with_defaults(:env => 'dev', :options => default_options[system]) + sh(django_admin(system, args.env, 'runserver', args.options)) + end + + # Per environment tasks + environments(system).each do |env| + desc "Attempt to import the settings file #{system}.envs.#{env} and report any errors" + task "#{system}:check_settings:#{env}" => :predjango do + sh("echo 'import #{system}.envs.#{env}' | #{django_admin(system, env, 'shell')}") + end + end +end + +desc "Reset the relational database used by django. WARNING: this will delete all of your existing users" +task :resetdb, [:env] do |t, args| + args.with_defaults(:env => 'dev') + sh(django_admin(:lms, args.env, 'syncdb')) + sh(django_admin(:lms, args.env, 'migrate')) +end + +desc "Update the relational database to the latest migration" +task :migrate, [:env] do |t, args| + args.with_defaults(:env => 'dev') + sh(django_admin(:lms, args.env, 'migrate')) +end + +task :runserver => :lms + +desc "Run django-admin against the specified system and environment" +task "django-admin", [:action, :system, :env, :options] do |t, args| + args.with_defaults(:env => 'dev', :system => 'lms', :options => '') + sh(django_admin(args.system, args.env, args.action, args.options)) +end + +desc "Set the staff bit for a user" +task :set_staff, [:user, :system, :env] do |t, args| + args.with_defaults(:env => 'dev', :system => 'lms', :options => '') + sh(django_admin(args.system, args.env, 'set_staff', args.user)) +end + +namespace :cms do + desc "Clone existing MongoDB based course" + task :clone do + + if ENV['SOURCE_LOC'] and ENV['DEST_LOC'] + sh(django_admin(:cms, :dev, :clone, ENV['SOURCE_LOC'], ENV['DEST_LOC'])) + else + raise "You must pass in a SOURCE_LOC and DEST_LOC parameters" + end + end + + desc "Delete existing MongoDB based course" + task :delete_course do + + if ENV['LOC'] and ENV['COMMIT'] + sh(django_admin(:cms, :dev, :delete_course, ENV['LOC'], ENV['COMMIT'])) + elsif ENV['LOC'] + sh(django_admin(:cms, :dev, :delete_course, ENV['LOC'])) + else + raise "You must pass in a LOC parameter" + end + end + + desc "Import course data within the given DATA_DIR variable" + task :import do + if ENV['DATA_DIR'] and ENV['COURSE_DIR'] + sh(django_admin(:cms, :dev, :import, ENV['DATA_DIR'], ENV['COURSE_DIR'])) + elsif ENV['DATA_DIR'] + sh(django_admin(:cms, :dev, :import, ENV['DATA_DIR'])) + else + raise "Please specify a DATA_DIR variable that point to your data directory.\n" + + "Example: \`rake cms:import DATA_DIR=../data\`" + end + end + + desc "Imports all the templates from the code pack" + task :update_templates do + sh(django_admin(:cms, :dev, :update_templates)) + end + + desc "Import course data within the given DATA_DIR variable" + task :xlint do + if ENV['DATA_DIR'] and ENV['COURSE_DIR'] + sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR'], ENV['COURSE_DIR'])) + elsif ENV['DATA_DIR'] + sh(django_admin(:cms, :dev, :xlint, ENV['DATA_DIR'])) + else + raise "Please specify a DATA_DIR variable that point to your data directory.\n" + + "Example: \`rake cms:import DATA_DIR=../data\`" + end + end + + desc "Export course data to a tar.gz file" + task :export do + if ENV['COURSE_ID'] and ENV['OUTPUT_PATH'] + sh(django_admin(:cms, :dev, :export, ENV['COURSE_ID'], ENV['OUTPUT_PATH'])) + else + raise "Please specify a COURSE_ID and OUTPUT_PATH.\n" + + "Example: \`rake cms:export COURSE_ID=MITx/12345/name OUTPUT_PATH=foo.tar.gz\`" + end + end +end \ No newline at end of file diff --git a/rakefiles/docs.rake b/rakefiles/docs.rake new file mode 100644 index 0000000000..f10fc80d59 --- /dev/null +++ b/rakefiles/docs.rake @@ -0,0 +1,34 @@ +require 'launchy' + +# --- Develop and public documentation --- +desc "Invoke sphinx 'make build' to generate docs." +task :builddocs, [:options] do |t, args| + if args.options == 'pub' + path = "doc/public" + else + path = "docs" + end + + Dir.chdir(path) do + sh('make html') + end +end + +desc "Show docs in browser (mac and ubuntu)." +task :showdocs, [:options] do |t, args| + if args.options == 'pub' + path = "doc/public" + else + path = "docs" + end + + Dir.chdir("#{path}/build/html") do + Launchy.open('index.html') + end +end + +desc "Build docs and show them in browser" +task :doc, [:options] => :builddocs do |t, args| + Rake::Task["showdocs"].invoke(args.options) +end +# --- Develop and public documentation --- diff --git a/rakefiles/helpers.rb b/rakefiles/helpers.rb new file mode 100644 index 0000000000..cd3d858e44 --- /dev/null +++ b/rakefiles/helpers.rb @@ -0,0 +1,60 @@ +require 'digest/md5' + + +def select_executable(*cmds) + cmds.find_all{ |cmd| system("which #{cmd} > /dev/null 2>&1") }[0] || fail("No executables found from #{cmds.join(', ')}") +end + +def django_admin(system, env, command, *args) + django_admin = ENV['DJANGO_ADMIN_PATH'] || select_executable('django-admin.py', 'django-admin') + return "#{django_admin} #{command} --traceback --settings=#{system}.envs.#{env} --pythonpath=. #{args.join(' ')}" +end + +def report_dir_path(dir) + return File.join(REPORT_DIR, dir.to_s) +end + +def when_changed(*files) + Rake::Task[PREREQS_MD5_DIR].invoke + cache_file = File.join(PREREQS_MD5_DIR, files.join('-').gsub(/\W+/, '-')) + '.md5' + digest = Digest::MD5.new() + Dir[*files].select{|file| File.file?(file)}.each do |file| + digest.file(file) + end + if !File.exists?(cache_file) or digest.hexdigest != File.read(cache_file) + yield + File.write(cache_file, digest.hexdigest) + end +end + +# Runs Process.spawn, and kills the process at the end of the rake process +# Expects the same arguments as Process.spawn +def background_process(*command) + pid = Process.spawn({}, *command, {:pgroup => true}) + + at_exit do + puts "Ending process and children" + pgid = Process.getpgid(pid) + begin + Timeout.timeout(5) do + puts "Terminating process group #{pgid}" + Process.kill(:SIGTERM, -pgid) + puts "Waiting on process group #{pgid}" + Process.wait(-pgid) + puts "Done waiting on process group #{pgid}" + end + rescue Timeout::Error + puts "Killing process group #{pgid}" + Process.kill(:SIGKILL, -pgid) + puts "Waiting on process group #{pgid}" + Process.wait(-pgid) + puts "Done waiting on process group #{pgid}" + end + end +end + +def environments(system) + Dir["#{system}/envs/**/*.py"].select{|file| ! (/__init__.py$/ =~ file)}.map do |env_file| + env_file.gsub("#{system}/envs/", '').gsub(/\.py/, '').gsub('/', '.') + end +end diff --git a/rakefiles/i18n.rake b/rakefiles/i18n.rake new file mode 100644 index 0000000000..e30c119e2e --- /dev/null +++ b/rakefiles/i18n.rake @@ -0,0 +1,73 @@ +# --- Internationalization tasks + +namespace :i18n do + + desc "Extract localizable strings from sources" + task :extract => "i18n:validate:gettext" do + sh(File.join(REPO_ROOT, "i18n", "extract.py")) + end + + desc "Compile localizable strings from sources. With optional flag 'extract', will extract strings first." + task :generate => "i18n:validate:gettext" do + if ARGV.last.downcase == 'extract' + Rake::Task["i18n:extract"].execute + end + sh(File.join(REPO_ROOT, "i18n", "generate.py")) + end + + desc "Simulate international translation by generating dummy strings corresponding to source strings." + task :dummy do + source_files = Dir["#{REPO_ROOT}/conf/locale/en/LC_MESSAGES/*.po"] + dummy_locale = 'fr' + cmd = File.join(REPO_ROOT, "i18n", "make_dummy.py") + for file in source_files do + sh("#{cmd} #{file} #{dummy_locale}") + end + end + + namespace :validate do + + desc "Make sure GNU gettext utilities are available" + task :gettext do + begin + select_executable('xgettext') + rescue + msg = "Cannot locate GNU gettext utilities, which are required by django for internationalization.\n" + msg += "(see https://docs.djangoproject.com/en/dev/topics/i18n/translation/#message-files)\n" + msg += "Try downloading them from http://www.gnu.org/software/gettext/" + abort(msg.red) + end + end + + desc "Make sure config file with username/password exists" + task :transifex_config do + config_file = "#{Dir.home}/.transifexrc" + if !File.file?(config_file) or File.size(config_file)==0 + msg ="Cannot connect to Transifex, config file is missing or empty: #{config_file}\n" + msg += "See http://help.transifex.com/features/client/#transifexrc" + abort(msg.red) + end + end + end + + namespace :transifex do + desc "Push source strings to Transifex for translation" + task :push => "i18n:validate:transifex_config" do + cmd = File.join(REPO_ROOT, "i18n", "transifex.py") + sh("#{cmd} push") + end + + desc "Pull translated strings from Transifex" + task :pull => "i18n:validate:transifex_config" do + cmd = File.join(REPO_ROOT, "i18n", "transifex.py") + sh("#{cmd} pull") + end + end + + desc "Run tests for the internationalization library" + task :test => "i18n:validate:gettext" do + test = File.join(REPO_ROOT, "i18n", "tests") + sh("nosetests #{test}") + end + +end diff --git a/rakefiles/jasmine.rake b/rakefiles/jasmine.rake new file mode 100644 index 0000000000..d9b3bee427 --- /dev/null +++ b/rakefiles/jasmine.rake @@ -0,0 +1,97 @@ +require 'colorize' +require 'erb' +require 'launchy' +require 'net/http' + + +def django_for_jasmine(system, django_reload) + if !django_reload + reload_arg = '--noreload' + end + + port = 10000 + rand(40000) + jasmine_url = "http://localhost:#{port}/_jasmine/" + + background_process(*django_admin(system, 'jasmine', 'runserver', '-v', '0', port.to_s, reload_arg).split(' ')) + + up = false + start_time = Time.now + until up do + if Time.now - start_time > 30 + abort "Timed out waiting for server to start to run jasmine tests" + end + begin + response = Net::HTTP.get_response(URI(jasmine_url)) + puts response.code + up = response.code == '200' + rescue => e + puts e.message + ensure + puts('Waiting server to start') + sleep(0.5) + end + end + yield jasmine_url +end + +def template_jasmine_runner(lib) + coffee_files = Dir["#{lib}/**/js/**/*.coffee", "common/static/coffee/src/**/*.coffee"] + if !coffee_files.empty? + sh("node_modules/.bin/coffee -c #{coffee_files.join(' ')}") + end + phantom_jasmine_path = File.expand_path("node_modules/phantom-jasmine") + common_js_root = File.expand_path("common/static/js") + common_coffee_root = File.expand_path("common/static/coffee/src") + + # Get arrays of spec and source files, ordered by how deep they are nested below the library + # (and then alphabetically) and expanded from a relative to an absolute path + spec_glob = File.join("#{lib}", "**", "spec", "**", "*.js") + src_glob = File.join("#{lib}", "**", "src", "**", "*.js") + js_specs = Dir[spec_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)} + js_source = Dir[src_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)} + + template = ERB.new(File.read("#{lib}/jasmine_test_runner.html.erb")) + template_output = "#{lib}/jasmine_test_runner.html" + File.open(template_output, 'w') do |f| + f.write(template.result(binding)) + end + yield File.expand_path(template_output) +end + +[:lms, :cms].each do |system| + desc "Open jasmine tests for #{system} in your default browser" + task "browse_jasmine_#{system}" => :assets do + django_for_jasmine(system, true) do |jasmine_url| + Launchy.open(jasmine_url) + puts "Press ENTER to terminate".red + $stdin.gets + end + end + + desc "Use phantomjs to run jasmine tests for #{system} from the console" + task "phantomjs_jasmine_#{system}" => :assets do + phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs' + django_for_jasmine(system, false) do |jasmine_url| + sh("#{phantomjs} node_modules/phantom-jasmine/lib/run_jasmine_test.coffee #{jasmine_url}") + end + end +end + +Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib| + desc "Open jasmine tests for #{lib} in your default browser" + task "browse_jasmine_#{lib}" do + template_jasmine_runner(lib) do |f| + sh("python -m webbrowser -t 'file://#{f}'") + puts "Press ENTER to terminate".red + $stdin.gets + end + end + + desc "Use phantomjs to run jasmine tests for #{lib} from the console" + task "phantomjs_jasmine_#{lib}" do + phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs' + template_jasmine_runner(lib) do |f| + sh("#{phantomjs} node_modules/phantom-jasmine/lib/run_jasmine_test.coffee #{f}") + end + end +end diff --git a/rakefiles/prereqs.rake b/rakefiles/prereqs.rake new file mode 100644 index 0000000000..7bd1940b4c --- /dev/null +++ b/rakefiles/prereqs.rake @@ -0,0 +1,39 @@ +require './rakefiles/helpers.rb' + + +PREREQS_MD5_DIR = ENV["PREREQ_CACHE_DIR"] || File.join(REPO_ROOT, '.prereqs_cache') + +CLOBBER.include(PREREQS_MD5_DIR) + +directory PREREQS_MD5_DIR + +desc "Install all prerequisites needed for the lms and cms" +task :install_prereqs => [:install_node_prereqs, :install_ruby_prereqs, :install_python_prereqs] + +desc "Install all node prerequisites for the lms and cms" +task :install_node_prereqs do + when_changed('package.json') do + sh('npm install') + end unless ENV['NO_PREREQ_INSTALL'] +end + +desc "Install all ruby prerequisites for the lms and cms" +task :install_ruby_prereqs do + when_changed('Gemfile') do + sh('bundle install') + end unless ENV['NO_PREREQ_INSTALL'] +end + +desc "Install all python prerequisites for the lms and cms" +task :install_python_prereqs do + when_changed('requirements/**') do + ENV['PIP_DOWNLOAD_CACHE'] ||= '.pip_download_cache' + sh('pip install --exists-action w -r requirements/base.txt') + sh('pip install --exists-action w -r requirements/post.txt') + # Check for private-requirements.txt: used to install our libs as working dirs, + # or personal-use tools. + if File.file?("requirements/private.txt") + sh('pip install -r requirements/private.txt') + end + end unless ENV['NO_PREREQ_INSTALL'] +end \ No newline at end of file diff --git a/rakefiles/quality.rake b/rakefiles/quality.rake new file mode 100644 index 0000000000..1d692c94cf --- /dev/null +++ b/rakefiles/quality.rake @@ -0,0 +1,31 @@ + +[:lms, :cms, :common].each do |system| + report_dir = report_dir_path(system) + directory report_dir + + desc "Run pep8 on all #{system} code" + task "pep8_#{system}" => report_dir do + sh("pep8 #{system} | tee #{report_dir}/pep8.report") + end + task :pep8 => "pep8_#{system}" + + desc "Run pylint on all #{system} code" + task "pylint_#{system}" => report_dir do + apps = Dir["#{system}/*.py", "#{system}/djangoapps/*", "#{system}/lib/*"].map do |app| + File.basename(app) + end.select do |app| + app !=~ /.pyc$/ + end.map do |app| + if app =~ /.py$/ + app.gsub('.py', '') + else + app + end + end + + pythonpath_prefix = "PYTHONPATH=#{system}:#{system}/djangoapps:#{system}/lib:common/djangoapps:common/lib" + sh("#{pythonpath_prefix} pylint --rcfile=.pylintrc -f parseable #{apps.join(' ')} | tee #{report_dir}/pylint.report") + end + task :pylint => "pylint_#{system}" + +end \ No newline at end of file diff --git a/rakefiles/tests.rake b/rakefiles/tests.rake new file mode 100644 index 0000000000..c3c7c72584 --- /dev/null +++ b/rakefiles/tests.rake @@ -0,0 +1,119 @@ + +# Set up the clean and clobber tasks +CLOBBER.include(REPORT_DIR, 'test_root/*_repo', 'test_root/staticfiles') + +$failed_tests = 0 + +def run_under_coverage(cmd, root) + cmd0, cmd_rest = cmd.split(" ", 2) + # We use "python -m coverage" so that the proper python will run the importable coverage + # rather than the coverage that OS path finds. + cmd = "python -m coverage run --rcfile=#{root}/.coveragerc `which #{cmd0}` #{cmd_rest}" + return cmd +end + +def run_tests(system, report_dir, stop_on_failure=true) + ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml") + dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"] + cmd = django_admin(system, :test, 'test', '--logging-clear-handlers', *dirs.each) + sh(run_under_coverage(cmd, system)) do |ok, res| + if !ok and stop_on_failure + abort "Test failed!" + end + $failed_tests += 1 unless ok + end +end + +directory REPORT_DIR + +task :clean_test_files do + sh("git clean -fqdx test_root") +end + +TEST_TASK_DIRS = [] + +[:lms, :cms].each do |system| + report_dir = report_dir_path(system) + + # Per System tasks + desc "Run all django tests on our djangoapps for the #{system}" + task "test_#{system}", [:stop_on_failure] => ["clean_test_files", :predjango, "#{system}:gather_assets:test", "fasttest_#{system}"] + + # Have a way to run the tests without running collectstatic -- useful when debugging without + # messing with static files. + task "fasttest_#{system}", [:stop_on_failure] => [report_dir, :install_prereqs, :predjango] do |t, args| + args.with_defaults(:stop_on_failure => 'true') + run_tests(system, report_dir, args.stop_on_failure) + end + + task :fasttest => "fasttest_#{system}" + + TEST_TASK_DIRS << system +end + +Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib| + task_name = "test_#{lib}" + + report_dir = report_dir_path(lib) + + desc "Run tests for common lib #{lib}" + task task_name => report_dir do + ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml") + cmd = "nosetests #{lib}" + sh(run_under_coverage(cmd, lib)) do |ok, res| + $failed_tests += 1 unless ok + end + end + TEST_TASK_DIRS << lib + + desc "Run tests for common lib #{lib} (without coverage)" + task "fasttest_#{lib}" do + sh("nosetests #{lib}") + end +end + +task :report_dirs + +TEST_TASK_DIRS.each do |dir| + report_dir = report_dir_path(dir) + directory report_dir + task :report_dirs => [REPORT_DIR, report_dir] +end + +task :test do + TEST_TASK_DIRS.each do |dir| + Rake::Task["test_#{dir}"].invoke(false) + end + + if $failed_tests > 0 + abort "Tests failed!" + end +end + +namespace :coverage do + desc "Build the html coverage reports" + task :html => :report_dirs do + TEST_TASK_DIRS.each do |dir| + report_dir = report_dir_path(dir) + + if !File.file?("#{report_dir}/.coverage") + next + end + + sh("coverage html --rcfile=#{dir}/.coveragerc") + end + end + + desc "Build the xml coverage reports" + task :xml => :report_dirs do + TEST_TASK_DIRS.each do |dir| + report_dir = report_dir_path(dir) + + if !File.file?("#{report_dir}/.coverage") + next + end + # Why doesn't the rcfile control the xml output file properly?? + sh("coverage xml -o #{report_dir}/coverage.xml --rcfile=#{dir}/.coveragerc") + end + end +end \ No newline at end of file