diff --git a/doc/testing.md b/doc/testing.md index 4d286b1bcc..c806b4e4ce 100644 --- a/doc/testing.md +++ b/doc/testing.md @@ -141,21 +141,36 @@ Very handy: if you uncomment the `pdb=1` line in `setup.cfg`, it will drop you i ### Running Javascript Unit Tests -These commands start a development server with jasmine testing enabled, and launch your default browser -pointing to those tests +To run all of the javascript unit tests, use - rake browse_jasmine_{lms,cms} + rake jasmine -To run the tests headless, you must install [phantomjs](http://phantomjs.org/download.html), then run: +If the `phantomjs` binary is on the path, or the `PHANTOMJS_PATH` environment variable is +set to point to it, then the tests will be run headless. Otherwise, they will be run in +your default browser - rake phantomjs_jasmine_{lms,cms} + export PATH=/path/to/phantomjs:$PATH + rake jasmine # Runs headless -If the `phantomjs` binary is not on the path, set the `PHANTOMJS_PATH` environment variable to point to it +or - PHANTOMJS_PATH=/path/to/phantomjs rake phantomjs_jasmine_{lms,cms} + PHANTOMJS_PATH=/path/to/phantomjs rake jasmine # Runs headless -Once you have run the `rake` command, your browser should open to -to `http://localhost/_jasmine/`, which displays the test results. +or + + rake jasmine # Runs in browser + +You can also force a run using phantomjs or the browser using the commands + + rake jasmine:browser # Runs in browser + rake jasmine:phantomjs # Runs headless + +You can run tests for a specific subsystems as well + + rake jasmine:lms # Runs all lms javascript unit tests using the default method + rake jasmine:cms:browser # Runs all cms javascript unit tests in the browser + +Use `rake -T` to get a list of all available subsystems **Troubleshooting**: If you get an error message while running the `rake` task, try running `bundle install` to install the required ruby gems. diff --git a/jenkins/test.sh b/jenkins/test.sh index 35be3a0121..8c4df9fac3 100755 --- a/jenkins/test.sh +++ b/jenkins/test.sh @@ -70,23 +70,11 @@ rake clobber rake pep8 > pep8.log || cat pep8.log rake pylint > pylint.log || cat pylint.log -TESTS_FAILED=0 - -# Run the python unit tests -rake test_cms || TESTS_FAILED=1 -rake test_lms || TESTS_FAILED=1 -rake test_common/lib/capa || TESTS_FAILED=1 -rake test_common/lib/xmodule || TESTS_FAILED=1 - -# Run the javascript unit tests -rake phantomjs_jasmine_lms || TESTS_FAILED=1 -rake phantomjs_jasmine_cms || TESTS_FAILED=1 -rake phantomjs_jasmine_common/lib/xmodule || TESTS_FAILED=1 -rake phantomjs_jasmine_common/static/coffee || TESTS_FAILED=1 +# Run the unit tests (use phantomjs for javascript unit tests) +rake test rake coverage:xml coverage:html -[ $TESTS_FAILED == '0' ] rake autodeploy_properties github_status state:success "passed" diff --git a/rakefiles/deprecated.rake b/rakefiles/deprecated.rake new file mode 100644 index 0000000000..00c1987bd5 --- /dev/null +++ b/rakefiles/deprecated.rake @@ -0,0 +1,23 @@ + +require 'colorize' + +def deprecated(deprecated, deprecated_by) + task deprecated do + puts("Task #{deprecated} has been deprecated. Use #{deprecated_by} instead. Waiting 5 seconds...".red) + sleep(5) + Rake::Task[deprecated_by].invoke + end +end + +[:lms, :cms].each do |system| + deprecated("browse_jasmine_#{system}", "jasmine:#{system}:browser") + deprecated("phantomjs_jasmine_#{system}", "jasmine:#{system}:phantomjs") +end + +Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib| + deprecated("browse_jasmine_#{lib}", "jasmine:#{lib}:browser") + deprecated("phantomjs_jasmine_#{lib}", "jasmine:#{lib}:phantomjs") +end + +deprecated("browse_jasmine_discussion", "jasmine:common/static/coffee:browser") +deprecated("phantomjs_jasmine_discussion", "jasmine:common/static/coffee:phantomjs") \ No newline at end of file diff --git a/rakefiles/helpers.rb b/rakefiles/helpers.rb index f344aa2042..4b10bef709 100644 --- a/rakefiles/helpers.rb +++ b/rakefiles/helpers.rb @@ -1,8 +1,12 @@ require 'digest/md5' +def find_executable(exec) + path = %x(which #{exec}).strip + $?.exitstatus == 0 ? path : nil +end def select_executable(*cmds) - cmds.find_all{ |cmd| system("which #{cmd} > /dev/null 2>&1") }[0] || fail("No executables found from #{cmds.join(', ')}") + cmds.find_all{ |cmd| !find_executable(cmd).nil? }[0] || fail("No executables found from #{cmds.join(', ')}") end def django_admin(system, env, command, *args) @@ -85,3 +89,31 @@ def environments(system) env_file.gsub("#{system}/envs/", '').gsub(/\.py/, '').gsub('/', '.') end end + +$failed_tests = 0 + +# Run sh on args. If TESTS_FAIL_FAST is set, then stop on the first shell failure. +# Otherwise, a final task will be added that will fail if any tests have failed +def test_sh(*args) + sh(*args) do |ok, res| + if ok + return + end + + if ENV['TESTS_FAIL_FAST'] + fail("Test failed!") + else + $failed_tests += 1 + end + end +end + +# Add a task after all other tasks that fails if any tests have failed +if !ENV['TESTS_FAIL_FAST'] + task :fail_tests do + fail("#{$failed_tests} tests failed!") if $failed_tests > 0 + end + + Rake.application.top_level_tasks << :fail_tests +end + diff --git a/rakefiles/jasmine.rake b/rakefiles/jasmine.rake index bd1c7e5d6c..ab3209c9ec 100644 --- a/rakefiles/jasmine.rake +++ b/rakefiles/jasmine.rake @@ -3,6 +3,11 @@ require 'erb' require 'launchy' require 'net/http' +PHANTOMJS_PATH = find_executable(ENV['PHANTOMJS_PATH'] || 'phantomjs') +PREFERRED_METHOD = PHANTOMJS_PATH.nil? ? 'browser' : 'phantomjs' +if PHANTOMJS_PATH.nil? + puts("phantomjs not found on path. Set $PHANTOMJS_PATH. Using browser for jasmine tests".blue) +end def django_for_jasmine(system, django_reload) if !django_reload @@ -35,18 +40,6 @@ def django_for_jasmine(system, django_reload) end def template_jasmine_runner(lib) - case lib - when /common\/lib\/.+/ - coffee_files = Dir["#{lib}/**/js/**/*.coffee", "common/static/coffee/src/**/*.coffee"] - when /common\/static\/coffee/ - coffee_files = Dir["#{lib}/**/*.coffee"] - else - puts('I do not know how to run jasmine tests for #{lib}') - exit - end - if !coffee_files.empty? - sh("node_modules/.bin/coffee -c #{coffee_files.join(' ')}") - end phantom_jasmine_path = File.expand_path("node_modules/phantom-jasmine") jasmine_reporters_path = File.expand_path("node_modules/jasmine-reporters") common_js_root = File.expand_path("common/static/js") @@ -54,8 +47,8 @@ def template_jasmine_runner(lib) # Get arrays of spec and source files, ordered by how deep they are nested below the library # (and then alphabetically) and expanded from a relative to an absolute path - spec_glob = File.join("#{lib}", "**", "spec", "**", "*.js") - src_glob = File.join("#{lib}", "**", "src", "**", "*.js") + spec_glob = File.join(lib, "**", "spec", "**", "*.js") + src_glob = File.join(lib, "**", "src", "**", "*.js") js_specs = Dir[spec_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)} js_source = Dir[src_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)} @@ -68,74 +61,90 @@ def template_jasmine_runner(lib) yield File.expand_path(template_output) end -def run_phantom_js(url) - phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs' - sh("#{phantomjs} node_modules/jasmine-reporters/test/phantomjs-testrunner.js #{url}") +def jasmine_browser(url, wait=10) + # Jitter starting the browser so that the tests don't all try and + # start the browser simultaneously + sleep(rand(3)) + sh("python -m webbrowser -t '#{url}'") + sleep(wait) end -# Open jasmine tests for :system in the default browser. The :env -# should (always?) be 'jasmine', but it's passed as an arg so that -# the :assets dependency gets it. -# -# This task should be invoked via the wrapper below, so we don't -# include a description to keep it from showing up in rake -T. -task :browse_jasmine, [:system, :env] => :assets do |t, args| - django_for_jasmine(args.system, true) do |jasmine_url| - Launchy.open(jasmine_url) - puts "Press ENTER to terminate".red - $stdin.gets - end -end - -# Use phantomjs to run jasmine tests from the console. The :env -# should (always?) be 'jasmine', but it's passed as an arg so that -# the :assets dependency gets it. -# -# This task should be invoked via the wrapper below, so we don't -# include a description to keep it from showing up in rake -T. -task :phantomjs_jasmine, [:system, :env] => :assets do |t, args| - django_for_jasmine(args.system, false) do |jasmine_url| - run_phantom_js(jasmine_url) - end +def jasmine_phantomjs(url) + fail("phantomjs not found. Add it to your path, or set $PHANTOMJS_PATH") if PHANTOMJS_PATH.nil? + test_sh("#{PHANTOMJS_PATH} node_modules/jasmine-reporters/test/phantomjs-testrunner.js #{url}") end # Wrapper tasks for the real browse_jasmine and phantomjs_jasmine # tasks above. These have a nicer UI since there's no arg passing. [:lms, :cms].each do |system| - desc "Open jasmine tests for #{system} in your default browser" - task "browse_jasmine_#{system}" do - Rake::Task[:browse_jasmine].invoke(system, 'jasmine') - end + namespace :jasmine do + namespace system do + desc "Open jasmine tests for #{system} in your default browser" + task :browser do + Rake::Task[:assets].invoke(system, 'jasmine') + django_for_jasmine(system, true) do |jasmine_url| + jasmine_browser(jasmine_url) + end + end - desc "Use phantomjs to run jasmine tests for #{system} from the console" - task "phantomjs_jasmine_#{system}" do - Rake::Task[:phantomjs_jasmine].invoke(system, 'jasmine') + desc "Use phantomjs to run jasmine tests for #{system} from the console" + task :phantomjs do + Rake::Task[:assets].invoke(system, 'jasmine') + phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs' + django_for_jasmine(system, false) do |jasmine_url| + jasmine_phantomjs(jasmine_url) + end + end + end + + desc "Run jasmine tests for #{system} using #{PREFERRED_METHOD}" + task system => "jasmine:#{system}:#{PREFERRED_METHOD}" + + task :phantomjs => "jasmine:#{system}:phantomjs" + multitask :browser => "jasmine:#{system}:browser" end end -STATIC_JASMINE_TESTS = Dir["common/lib/*"].select{|lib| File.directory?(lib)} -STATIC_JASMINE_TESTS << 'common/static/coffee' +static_js_dirs = Dir["common/lib/*"].select{|lib| File.directory?(lib)} +static_js_dirs << 'common/static/coffee' +static_js_dirs.select!{|lib| !Dir["#{lib}/**/spec"].empty?} -STATIC_JASMINE_TESTS.each do |lib| - desc "Open jasmine tests for #{lib} in your default browser" - task "browse_jasmine_#{lib}" do - template_jasmine_runner(lib) do |f| - sh("python -m webbrowser -t 'file://#{f}'") - puts "Press ENTER to terminate".red - $stdin.gets - end - end +static_js_dirs.each do |dir| + namespace :jasmine do + namespace dir do + desc "Open jasmine tests for #{dir} in your default browser" + task :browser do + # We need to use either CMS or LMS to preprocess files. Use LMS by default + Rake::Task['assets:coffee'].invoke('lms', 'jasmine') + template_jasmine_runner(dir) do |f| + jasmine_browser("file://#{f}") + end + end - desc "Use phantomjs to run jasmine tests for #{lib} from the console" - task "phantomjs_jasmine_#{lib}" do - template_jasmine_runner(lib) do |f| - run_phantom_js(f) + desc "Use phantomjs to run jasmine tests for #{dir} from the console" + task :phantomjs do + # We need to use either CMS or LMS to preprocess files. Use LMS by default + Rake::Task[:assets].invoke('lms', 'jasmine') + template_jasmine_runner(dir) do |f| + jasmine_phantomjs(f) + end + end end + + desc "Run jasmine tests for #{dir} using #{PREFERRED_METHOD}" + task dir => "jasmine:#{dir}:#{PREFERRED_METHOD}" + + task :phantomjs => "jasmine:#{dir}:phantomjs" + multitask :browser => "jasmine:#{dir}:browser" end end -desc "Open jasmine tests for discussion in your default browser" -task "browse_jasmine_discussion" => "browse_jasmine_common/static/coffee" +desc "Run all jasmine tests using #{PREFERRED_METHOD}" +task :jasmine => "jasmine:#{PREFERRED_METHOD}" -desc "Use phantomjs to run jasmine tests for discussion from the console" -task "phantomjs_jasmine_discussion" => "phantomjs_jasmine_common/static/coffee" +['phantomjs', 'browser'].each do |method| + desc "Run all jasmine tests using #{method}" + task "jasmine:#{method}" +end + +task :test => :jasmine diff --git a/rakefiles/tests.rake b/rakefiles/tests.rake index 448a482f04..592f777b39 100644 --- a/rakefiles/tests.rake +++ b/rakefiles/tests.rake @@ -1,9 +1,6 @@ - # Set up the clean and clobber tasks CLOBBER.include(REPORT_DIR, 'test_root/*_repo', 'test_root/staticfiles') -$failed_tests = 0 - def run_under_coverage(cmd, root) cmd0, cmd_rest = cmd.split(" ", 2) # We use "python -m coverage" so that the proper python will run the importable coverage @@ -17,12 +14,7 @@ def run_tests(system, report_dir, test_id=nil, stop_on_failure=true) dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"] test_id = dirs.join(' ') if test_id.nil? or test_id == '' cmd = django_admin(system, :test, 'test', '--logging-clear-handlers', test_id) - sh(run_under_coverage(cmd, system)) do |ok, res| - if !ok and stop_on_failure - abort "Test failed!" - end - $failed_tests += 1 unless ok - end + test_sh(run_under_coverage(cmd, system)) end def run_acceptance_tests(system, report_dir, harvest_args) @@ -38,7 +30,7 @@ def run_acceptance_tests(system, report_dir, harvest_args) end sh(django_admin(system, 'acceptance', 'syncdb', '--noinput')) sh(django_admin(system, 'acceptance', 'migrate', '--noinput')) - sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--tag -skip', harvest_args)) + test_sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--tag -skip', harvest_args)) end @@ -55,13 +47,13 @@ TEST_TASK_DIRS = [] # Per System tasks desc "Run all django tests on our djangoapps for the #{system}" - task "test_#{system}", [:test_id, :stop_on_failure] => ["clean_test_files", :predjango, "#{system}:gather_assets:test", "fasttest_#{system}"] + task "test_#{system}", [:test_id] => ["clean_test_files", :predjango, "#{system}:gather_assets:test", "fasttest_#{system}"] # Have a way to run the tests without running collectstatic -- useful when debugging without # messing with static files. - task "fasttest_#{system}", [:test_id, :stop_on_failure] => [report_dir, :install_prereqs, :predjango] do |t, args| - args.with_defaults(:stop_on_failure => 'true', :test_id => nil) - run_tests(system, report_dir, args.test_id, args.stop_on_failure) + task "fasttest_#{system}", [:test_id] => [report_dir, :install_prereqs, :predjango] do |t, args| + args.with_defaults(:test_id => nil) + run_tests(system, report_dir, args.test_id) end # Run acceptance tests @@ -89,9 +81,7 @@ Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib| task task_name => report_dir do ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml") cmd = "nosetests #{lib}" - sh(run_under_coverage(cmd, lib)) do |ok, res| - $failed_tests += 1 unless ok - end + test_sh(run_under_coverage(cmd, lib)) end TEST_TASK_DIRS << lib @@ -107,17 +97,11 @@ TEST_TASK_DIRS.each do |dir| report_dir = report_dir_path(dir) directory report_dir task :report_dirs => [REPORT_DIR, report_dir] + task :test => "test_#{dir}" end -task :test do - TEST_TASK_DIRS.each do |dir| - Rake::Task["test_#{dir}"].invoke(nil, false) - end - - if $failed_tests > 0 - abort "Tests failed!" - end -end +desc "Run all tests" +task :test namespace :coverage do desc "Build the html coverage reports"