diff --git a/rakefile b/rakefile new file mode 100644 index 0000000000..053abf56a8 --- /dev/null +++ b/rakefile @@ -0,0 +1,229 @@ +require 'rake/clean' +require 'tempfile' + +# Build Constants +REPO_ROOT = File.dirname(__FILE__) +BUILD_DIR = File.join(REPO_ROOT, "build") +REPORT_DIR = File.join(REPO_ROOT, "reports") +LMS_REPORT_DIR = File.join(REPORT_DIR, "lms") + +# Packaging constants +DEPLOY_DIR = "/opt/wwc" +PACKAGE_NAME = "mitx" +LINK_PATH = "/opt/wwc/mitx" +PKG_VERSION = "0.1" +COMMIT = (ENV["GIT_COMMIT"] || `git rev-parse HEAD`).chomp()[0, 10] +BRANCH = (ENV["GIT_BRANCH"] || `git symbolic-ref -q HEAD`).chomp().gsub('refs/heads/', '').gsub('origin/', '') +BUILD_NUMBER = (ENV["BUILD_NUMBER"] || "dev").chomp() + +if BRANCH == "master" + DEPLOY_NAME = "#{PACKAGE_NAME}-#{BUILD_NUMBER}-#{COMMIT}" +else + DEPLOY_NAME = "#{PACKAGE_NAME}-#{BRANCH}-#{BUILD_NUMBER}-#{COMMIT}" +end +PACKAGE_REPO = "packages@gp.mitx.mit.edu:/opt/pkgrepo.incoming" + +NORMALIZED_DEPLOY_NAME = DEPLOY_NAME.downcase().gsub(/[_\/]/, '-') +INSTALL_DIR_PATH = File.join(DEPLOY_DIR, NORMALIZED_DEPLOY_NAME) +# Set up the clean and clobber tasks +CLOBBER.include(BUILD_DIR, REPORT_DIR, 'cover*', '.coverage', 'test_root/*_repo', 'test_root/staticfiles') +CLEAN.include("#{BUILD_DIR}/*.deb", "#{BUILD_DIR}/util") + +def select_executable(*cmds) + cmds.find_all{ |cmd| system("which #{cmd} > /dev/null 2>&1") }[0] || fail("No executables found from #{cmds.join(', ')}") +end + +def django_admin(system, env, command, *args) + django_admin = ENV['DJANGO_ADMIN_PATH'] || select_executable('django-admin.py', 'django-admin') + return "#{django_admin} #{command} --settings=#{system}.envs.#{env} --pythonpath=. #{args.join(' ')}" +end + +task :default => [:test, :pep8, :pylint] + +directory REPORT_DIR + +default_options = { + :lms => '8000', + :cms => '8001', +} + +task :predjango do + sh("find . -type f -name *.pyc -delete") + sh('pip install -e common/lib/xmodule') + sh('git submodule update --init') +end + +task :clean_test_files do + sh("git clean -fdx test_root") +end + +[:lms, :cms, :common].each do |system| + report_dir = File.join(REPORT_DIR, system.to_s) + directory report_dir + + desc "Run pep8 on all #{system} code" + task "pep8_#{system}" => report_dir do + sh("pep8 --ignore=E501 #{system}/djangoapps #{system}/lib | tee #{report_dir}/pep8.report") + end + task :pep8 => "pep8_#{system}" + + desc "Run pylint on all #{system} code" + task "pylint_#{system}" => report_dir do + Dir["#{system}/djangoapps/*", "#{system}/lib/*"].each do |app| + if File.exists? "#{app}/setup.py" + pythonpath_prefix = "PYTHONPATH=#{app}" + else + pythonpath_prefix = "PYTHONPATH=#{File.dirname(app)}" + end + app = File.basename(app) + sh("#{pythonpath_prefix} pylint --rcfile=.pylintrc -f parseable #{app} | tee #{report_dir}/#{app}.pylint.report") + end + end + task :pylint => "pylint_#{system}" +end + +$failed_tests = 0 + +def run_tests(system, report_dir, stop_on_failure=true) + ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml") + ENV['NOSE_COVER_HTML_DIR'] = File.join(report_dir, "cover") + dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"] + sh(django_admin(system, :test, 'test', *dirs.each)) do |ok, res| + if !ok and stop_on_failure + abort "Test failed!" + end + $failed_tests += 1 unless ok + end +end + +TEST_TASKS = [] + +[:lms, :cms].each do |system| + report_dir = File.join(REPORT_DIR, system.to_s) + directory report_dir + + # Per System tasks + desc "Run all django tests on our djangoapps for the #{system}" + task "test_#{system}", [:stop_on_failure] => ["clean_test_files", "#{system}:collectstatic:test", "fasttest_#{system}"] + + # Have a way to run the tests without running collectstatic -- useful when debugging without + # messing with static files. + task "fasttest_#{system}", [:stop_on_failure] => [report_dir, :predjango] do |t, args| + args.with_defaults(:stop_on_failure => 'true') + run_tests(system, report_dir, args.stop_on_failure) + end + + TEST_TASKS << "test_#{system}" + + desc <<-desc + Start the #{system} locally with the specified environment (defaults to dev). + Other useful environments are devplus (for dev testing with a real local database) + desc + task system, [:env, :options] => [:predjango] do |t, args| + args.with_defaults(:env => 'dev', :options => default_options[system]) + sh(django_admin(system, args.env, 'runserver', args.options)) + end + + # Per environment tasks + Dir["#{system}/envs/*.py"].each do |env_file| + env = File.basename(env_file).gsub(/\.py/, '') + desc "Attempt to import the settings file #{system}.envs.#{env} and report any errors" + task "#{system}:check_settings:#{env}" => :predjango do + sh("echo 'import #{system}.envs.#{env}' | #{django_admin(system, env, 'shell')}") + end + + desc "Run collectstatic in the specified environment" + task "#{system}:collectstatic:#{env}" => :predjango do + sh("#{django_admin(system, env, 'collectstatic', '--noinput')}") + end + end +end + +Dir["common/lib/*"].each do |lib| + task_name = "test_#{lib}" + + report_dir = File.join(REPORT_DIR, task_name.gsub('/', '_')) + directory report_dir + + desc "Run tests for common lib #{lib}" + task task_name => report_dir do + ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml") + sh("nosetests #{lib} --cover-erase --with-xunit --with-xcoverage --cover-html --cover-inclusive --cover-package #{File.basename(lib)} --cover-html-dir #{File.join(report_dir, "cover")}") + end + TEST_TASKS << task_name +end + +task :test do + TEST_TASKS.each do |task| + Rake::Task[task].invoke(false) + end + + if $failed_tests > 0 + abort "Tests failed!" + end +end + +task :runserver => :lms + +desc "Run django-admin against the specified system and environment" +task "django-admin", [:action, :system, :env, :options] => [:predjango] do |t, args| + args.with_defaults(:env => 'dev', :system => 'lms', :options => '') + sh(django_admin(args.system, args.env, args.action, args.options)) +end + +task :package do + FileUtils.mkdir_p(BUILD_DIR) + + Dir.chdir(BUILD_DIR) do + afterremove = Tempfile.new('afterremove') + afterremove.write <<-AFTERREMOVE.gsub(/^\s*/, '') + #! /bin/bash + set -e + set -x + + # to be a little safer this rm is executed + # as the makeitso user + + if [[ -d "#{INSTALL_DIR_PATH}" ]]; then + sudo rm -rf "#{INSTALL_DIR_PATH}" + fi + + AFTERREMOVE + afterremove.close() + FileUtils.chmod(0755, afterremove.path) + + args = ["fakeroot", "fpm", "-s", "dir", "-t", "deb", + "--after-remove=#{afterremove.path}", + "--prefix=#{INSTALL_DIR_PATH}", + "--exclude=**/build/**", + "--exclude=**/rakefile", + "--exclude=**/.git/**", + "--exclude=**/*.pyc", + "--exclude=**/reports/**", + "--exclude=**/test_root/**", + "--exclude=**/.coverage/**", + "-C", "#{REPO_ROOT}", + "--provides=#{PACKAGE_NAME}", + "--name=#{NORMALIZED_DEPLOY_NAME}", + "--version=#{PKG_VERSION}", + "-a", "all", + "."] + system(*args) || raise("fpm failed to build the .deb") + end +end + +task :publish => :package do + sh("scp #{BUILD_DIR}/#{NORMALIZED_DEPLOY_NAME}_#{PKG_VERSION}*.deb #{PACKAGE_REPO}") +end + +namespace :cms do + desc "Import course data within the given DATA_DIR variable" + task :import do + if ENV['DATA_DIR'] + sh(django_admin(:cms, :dev, :import, ENV['DATA_DIR'])) + else + raise "Please specify a DATA_DIR variable that point to your data directory.\n" + + "Example: \`rake cms:import DATA_DIR=../data\`" + end + end +end