From 0bf7c71ec2880dce39df9f1a5710d5e78a63d5c7 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Wed, 5 Jun 2013 13:05:00 -0400 Subject: [PATCH] Run all tests on jenkins We used to specify specific rake test tasks so that we could run all of them even if early ones failed. However, that meant that as new tasks were added, they weren't being run on jenkins. Now, there is a facility in the rake scripts so that tests can run using the test_sh function, which will delay failure until the end of the rake run, unless the TESTS_FAIL_FAST environment variable is set. Furthermore, this reorganizes the jasmine test tasks so that we can run those as part of `rake test` as well. --- doc/testing.md | 33 ++++++--- jenkins/test.sh | 16 +---- rakefiles/deprecated.rake | 23 ++++++ rakefiles/helpers.rb | 34 ++++++++- rakefiles/jasmine.rake | 143 ++++++++++++++++++++------------------ rakefiles/tests.rake | 36 +++------- 6 files changed, 168 insertions(+), 117 deletions(-) create mode 100644 rakefiles/deprecated.rake diff --git a/doc/testing.md b/doc/testing.md index 4d286b1bcc..c806b4e4ce 100644 --- a/doc/testing.md +++ b/doc/testing.md @@ -141,21 +141,36 @@ Very handy: if you uncomment the `pdb=1` line in `setup.cfg`, it will drop you i ### Running Javascript Unit Tests -These commands start a development server with jasmine testing enabled, and launch your default browser -pointing to those tests +To run all of the javascript unit tests, use - rake browse_jasmine_{lms,cms} + rake jasmine -To run the tests headless, you must install [phantomjs](http://phantomjs.org/download.html), then run: +If the `phantomjs` binary is on the path, or the `PHANTOMJS_PATH` environment variable is +set to point to it, then the tests will be run headless. Otherwise, they will be run in +your default browser - rake phantomjs_jasmine_{lms,cms} + export PATH=/path/to/phantomjs:$PATH + rake jasmine # Runs headless -If the `phantomjs` binary is not on the path, set the `PHANTOMJS_PATH` environment variable to point to it +or - PHANTOMJS_PATH=/path/to/phantomjs rake phantomjs_jasmine_{lms,cms} + PHANTOMJS_PATH=/path/to/phantomjs rake jasmine # Runs headless -Once you have run the `rake` command, your browser should open to -to `http://localhost/_jasmine/`, which displays the test results. +or + + rake jasmine # Runs in browser + +You can also force a run using phantomjs or the browser using the commands + + rake jasmine:browser # Runs in browser + rake jasmine:phantomjs # Runs headless + +You can run tests for a specific subsystems as well + + rake jasmine:lms # Runs all lms javascript unit tests using the default method + rake jasmine:cms:browser # Runs all cms javascript unit tests in the browser + +Use `rake -T` to get a list of all available subsystems **Troubleshooting**: If you get an error message while running the `rake` task, try running `bundle install` to install the required ruby gems. diff --git a/jenkins/test.sh b/jenkins/test.sh index 35be3a0121..8c4df9fac3 100755 --- a/jenkins/test.sh +++ b/jenkins/test.sh @@ -70,23 +70,11 @@ rake clobber rake pep8 > pep8.log || cat pep8.log rake pylint > pylint.log || cat pylint.log -TESTS_FAILED=0 - -# Run the python unit tests -rake test_cms || TESTS_FAILED=1 -rake test_lms || TESTS_FAILED=1 -rake test_common/lib/capa || TESTS_FAILED=1 -rake test_common/lib/xmodule || TESTS_FAILED=1 - -# Run the javascript unit tests -rake phantomjs_jasmine_lms || TESTS_FAILED=1 -rake phantomjs_jasmine_cms || TESTS_FAILED=1 -rake phantomjs_jasmine_common/lib/xmodule || TESTS_FAILED=1 -rake phantomjs_jasmine_common/static/coffee || TESTS_FAILED=1 +# Run the unit tests (use phantomjs for javascript unit tests) +rake test rake coverage:xml coverage:html -[ $TESTS_FAILED == '0' ] rake autodeploy_properties github_status state:success "passed" diff --git a/rakefiles/deprecated.rake b/rakefiles/deprecated.rake new file mode 100644 index 0000000000..00c1987bd5 --- /dev/null +++ b/rakefiles/deprecated.rake @@ -0,0 +1,23 @@ + +require 'colorize' + +def deprecated(deprecated, deprecated_by) + task deprecated do + puts("Task #{deprecated} has been deprecated. Use #{deprecated_by} instead. Waiting 5 seconds...".red) + sleep(5) + Rake::Task[deprecated_by].invoke + end +end + +[:lms, :cms].each do |system| + deprecated("browse_jasmine_#{system}", "jasmine:#{system}:browser") + deprecated("phantomjs_jasmine_#{system}", "jasmine:#{system}:phantomjs") +end + +Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib| + deprecated("browse_jasmine_#{lib}", "jasmine:#{lib}:browser") + deprecated("phantomjs_jasmine_#{lib}", "jasmine:#{lib}:phantomjs") +end + +deprecated("browse_jasmine_discussion", "jasmine:common/static/coffee:browser") +deprecated("phantomjs_jasmine_discussion", "jasmine:common/static/coffee:phantomjs") \ No newline at end of file diff --git a/rakefiles/helpers.rb b/rakefiles/helpers.rb index f344aa2042..4b10bef709 100644 --- a/rakefiles/helpers.rb +++ b/rakefiles/helpers.rb @@ -1,8 +1,12 @@ require 'digest/md5' +def find_executable(exec) + path = %x(which #{exec}).strip + $?.exitstatus == 0 ? path : nil +end def select_executable(*cmds) - cmds.find_all{ |cmd| system("which #{cmd} > /dev/null 2>&1") }[0] || fail("No executables found from #{cmds.join(', ')}") + cmds.find_all{ |cmd| !find_executable(cmd).nil? }[0] || fail("No executables found from #{cmds.join(', ')}") end def django_admin(system, env, command, *args) @@ -85,3 +89,31 @@ def environments(system) env_file.gsub("#{system}/envs/", '').gsub(/\.py/, '').gsub('/', '.') end end + +$failed_tests = 0 + +# Run sh on args. If TESTS_FAIL_FAST is set, then stop on the first shell failure. +# Otherwise, a final task will be added that will fail if any tests have failed +def test_sh(*args) + sh(*args) do |ok, res| + if ok + return + end + + if ENV['TESTS_FAIL_FAST'] + fail("Test failed!") + else + $failed_tests += 1 + end + end +end + +# Add a task after all other tasks that fails if any tests have failed +if !ENV['TESTS_FAIL_FAST'] + task :fail_tests do + fail("#{$failed_tests} tests failed!") if $failed_tests > 0 + end + + Rake.application.top_level_tasks << :fail_tests +end + diff --git a/rakefiles/jasmine.rake b/rakefiles/jasmine.rake index bd1c7e5d6c..ab3209c9ec 100644 --- a/rakefiles/jasmine.rake +++ b/rakefiles/jasmine.rake @@ -3,6 +3,11 @@ require 'erb' require 'launchy' require 'net/http' +PHANTOMJS_PATH = find_executable(ENV['PHANTOMJS_PATH'] || 'phantomjs') +PREFERRED_METHOD = PHANTOMJS_PATH.nil? ? 'browser' : 'phantomjs' +if PHANTOMJS_PATH.nil? + puts("phantomjs not found on path. Set $PHANTOMJS_PATH. Using browser for jasmine tests".blue) +end def django_for_jasmine(system, django_reload) if !django_reload @@ -35,18 +40,6 @@ def django_for_jasmine(system, django_reload) end def template_jasmine_runner(lib) - case lib - when /common\/lib\/.+/ - coffee_files = Dir["#{lib}/**/js/**/*.coffee", "common/static/coffee/src/**/*.coffee"] - when /common\/static\/coffee/ - coffee_files = Dir["#{lib}/**/*.coffee"] - else - puts('I do not know how to run jasmine tests for #{lib}') - exit - end - if !coffee_files.empty? - sh("node_modules/.bin/coffee -c #{coffee_files.join(' ')}") - end phantom_jasmine_path = File.expand_path("node_modules/phantom-jasmine") jasmine_reporters_path = File.expand_path("node_modules/jasmine-reporters") common_js_root = File.expand_path("common/static/js") @@ -54,8 +47,8 @@ def template_jasmine_runner(lib) # Get arrays of spec and source files, ordered by how deep they are nested below the library # (and then alphabetically) and expanded from a relative to an absolute path - spec_glob = File.join("#{lib}", "**", "spec", "**", "*.js") - src_glob = File.join("#{lib}", "**", "src", "**", "*.js") + spec_glob = File.join(lib, "**", "spec", "**", "*.js") + src_glob = File.join(lib, "**", "src", "**", "*.js") js_specs = Dir[spec_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)} js_source = Dir[src_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)} @@ -68,74 +61,90 @@ def template_jasmine_runner(lib) yield File.expand_path(template_output) end -def run_phantom_js(url) - phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs' - sh("#{phantomjs} node_modules/jasmine-reporters/test/phantomjs-testrunner.js #{url}") +def jasmine_browser(url, wait=10) + # Jitter starting the browser so that the tests don't all try and + # start the browser simultaneously + sleep(rand(3)) + sh("python -m webbrowser -t '#{url}'") + sleep(wait) end -# Open jasmine tests for :system in the default browser. The :env -# should (always?) be 'jasmine', but it's passed as an arg so that -# the :assets dependency gets it. -# -# This task should be invoked via the wrapper below, so we don't -# include a description to keep it from showing up in rake -T. -task :browse_jasmine, [:system, :env] => :assets do |t, args| - django_for_jasmine(args.system, true) do |jasmine_url| - Launchy.open(jasmine_url) - puts "Press ENTER to terminate".red - $stdin.gets - end -end - -# Use phantomjs to run jasmine tests from the console. The :env -# should (always?) be 'jasmine', but it's passed as an arg so that -# the :assets dependency gets it. -# -# This task should be invoked via the wrapper below, so we don't -# include a description to keep it from showing up in rake -T. -task :phantomjs_jasmine, [:system, :env] => :assets do |t, args| - django_for_jasmine(args.system, false) do |jasmine_url| - run_phantom_js(jasmine_url) - end +def jasmine_phantomjs(url) + fail("phantomjs not found. Add it to your path, or set $PHANTOMJS_PATH") if PHANTOMJS_PATH.nil? + test_sh("#{PHANTOMJS_PATH} node_modules/jasmine-reporters/test/phantomjs-testrunner.js #{url}") end # Wrapper tasks for the real browse_jasmine and phantomjs_jasmine # tasks above. These have a nicer UI since there's no arg passing. [:lms, :cms].each do |system| - desc "Open jasmine tests for #{system} in your default browser" - task "browse_jasmine_#{system}" do - Rake::Task[:browse_jasmine].invoke(system, 'jasmine') - end + namespace :jasmine do + namespace system do + desc "Open jasmine tests for #{system} in your default browser" + task :browser do + Rake::Task[:assets].invoke(system, 'jasmine') + django_for_jasmine(system, true) do |jasmine_url| + jasmine_browser(jasmine_url) + end + end - desc "Use phantomjs to run jasmine tests for #{system} from the console" - task "phantomjs_jasmine_#{system}" do - Rake::Task[:phantomjs_jasmine].invoke(system, 'jasmine') + desc "Use phantomjs to run jasmine tests for #{system} from the console" + task :phantomjs do + Rake::Task[:assets].invoke(system, 'jasmine') + phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs' + django_for_jasmine(system, false) do |jasmine_url| + jasmine_phantomjs(jasmine_url) + end + end + end + + desc "Run jasmine tests for #{system} using #{PREFERRED_METHOD}" + task system => "jasmine:#{system}:#{PREFERRED_METHOD}" + + task :phantomjs => "jasmine:#{system}:phantomjs" + multitask :browser => "jasmine:#{system}:browser" end end -STATIC_JASMINE_TESTS = Dir["common/lib/*"].select{|lib| File.directory?(lib)} -STATIC_JASMINE_TESTS << 'common/static/coffee' +static_js_dirs = Dir["common/lib/*"].select{|lib| File.directory?(lib)} +static_js_dirs << 'common/static/coffee' +static_js_dirs.select!{|lib| !Dir["#{lib}/**/spec"].empty?} -STATIC_JASMINE_TESTS.each do |lib| - desc "Open jasmine tests for #{lib} in your default browser" - task "browse_jasmine_#{lib}" do - template_jasmine_runner(lib) do |f| - sh("python -m webbrowser -t 'file://#{f}'") - puts "Press ENTER to terminate".red - $stdin.gets - end - end +static_js_dirs.each do |dir| + namespace :jasmine do + namespace dir do + desc "Open jasmine tests for #{dir} in your default browser" + task :browser do + # We need to use either CMS or LMS to preprocess files. Use LMS by default + Rake::Task['assets:coffee'].invoke('lms', 'jasmine') + template_jasmine_runner(dir) do |f| + jasmine_browser("file://#{f}") + end + end - desc "Use phantomjs to run jasmine tests for #{lib} from the console" - task "phantomjs_jasmine_#{lib}" do - template_jasmine_runner(lib) do |f| - run_phantom_js(f) + desc "Use phantomjs to run jasmine tests for #{dir} from the console" + task :phantomjs do + # We need to use either CMS or LMS to preprocess files. Use LMS by default + Rake::Task[:assets].invoke('lms', 'jasmine') + template_jasmine_runner(dir) do |f| + jasmine_phantomjs(f) + end + end end + + desc "Run jasmine tests for #{dir} using #{PREFERRED_METHOD}" + task dir => "jasmine:#{dir}:#{PREFERRED_METHOD}" + + task :phantomjs => "jasmine:#{dir}:phantomjs" + multitask :browser => "jasmine:#{dir}:browser" end end -desc "Open jasmine tests for discussion in your default browser" -task "browse_jasmine_discussion" => "browse_jasmine_common/static/coffee" +desc "Run all jasmine tests using #{PREFERRED_METHOD}" +task :jasmine => "jasmine:#{PREFERRED_METHOD}" -desc "Use phantomjs to run jasmine tests for discussion from the console" -task "phantomjs_jasmine_discussion" => "phantomjs_jasmine_common/static/coffee" +['phantomjs', 'browser'].each do |method| + desc "Run all jasmine tests using #{method}" + task "jasmine:#{method}" +end + +task :test => :jasmine diff --git a/rakefiles/tests.rake b/rakefiles/tests.rake index 448a482f04..592f777b39 100644 --- a/rakefiles/tests.rake +++ b/rakefiles/tests.rake @@ -1,9 +1,6 @@ - # Set up the clean and clobber tasks CLOBBER.include(REPORT_DIR, 'test_root/*_repo', 'test_root/staticfiles') -$failed_tests = 0 - def run_under_coverage(cmd, root) cmd0, cmd_rest = cmd.split(" ", 2) # We use "python -m coverage" so that the proper python will run the importable coverage @@ -17,12 +14,7 @@ def run_tests(system, report_dir, test_id=nil, stop_on_failure=true) dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"] test_id = dirs.join(' ') if test_id.nil? or test_id == '' cmd = django_admin(system, :test, 'test', '--logging-clear-handlers', test_id) - sh(run_under_coverage(cmd, system)) do |ok, res| - if !ok and stop_on_failure - abort "Test failed!" - end - $failed_tests += 1 unless ok - end + test_sh(run_under_coverage(cmd, system)) end def run_acceptance_tests(system, report_dir, harvest_args) @@ -38,7 +30,7 @@ def run_acceptance_tests(system, report_dir, harvest_args) end sh(django_admin(system, 'acceptance', 'syncdb', '--noinput')) sh(django_admin(system, 'acceptance', 'migrate', '--noinput')) - sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--tag -skip', harvest_args)) + test_sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--tag -skip', harvest_args)) end @@ -55,13 +47,13 @@ TEST_TASK_DIRS = [] # Per System tasks desc "Run all django tests on our djangoapps for the #{system}" - task "test_#{system}", [:test_id, :stop_on_failure] => ["clean_test_files", :predjango, "#{system}:gather_assets:test", "fasttest_#{system}"] + task "test_#{system}", [:test_id] => ["clean_test_files", :predjango, "#{system}:gather_assets:test", "fasttest_#{system}"] # Have a way to run the tests without running collectstatic -- useful when debugging without # messing with static files. - task "fasttest_#{system}", [:test_id, :stop_on_failure] => [report_dir, :install_prereqs, :predjango] do |t, args| - args.with_defaults(:stop_on_failure => 'true', :test_id => nil) - run_tests(system, report_dir, args.test_id, args.stop_on_failure) + task "fasttest_#{system}", [:test_id] => [report_dir, :install_prereqs, :predjango] do |t, args| + args.with_defaults(:test_id => nil) + run_tests(system, report_dir, args.test_id) end # Run acceptance tests @@ -89,9 +81,7 @@ Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib| task task_name => report_dir do ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml") cmd = "nosetests #{lib}" - sh(run_under_coverage(cmd, lib)) do |ok, res| - $failed_tests += 1 unless ok - end + test_sh(run_under_coverage(cmd, lib)) end TEST_TASK_DIRS << lib @@ -107,17 +97,11 @@ TEST_TASK_DIRS.each do |dir| report_dir = report_dir_path(dir) directory report_dir task :report_dirs => [REPORT_DIR, report_dir] + task :test => "test_#{dir}" end -task :test do - TEST_TASK_DIRS.each do |dir| - Rake::Task["test_#{dir}"].invoke(nil, false) - end - - if $failed_tests > 0 - abort "Tests failed!" - end -end +desc "Run all tests" +task :test namespace :coverage do desc "Build the html coverage reports"