Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run all tests on jenkins

Browse Source 
We used to specify specific rake test tasks so that we could run all of
them even if early ones failed. However, that meant that as new tasks
were added, they weren't being run on jenkins.

Now, there is a facility in the rake scripts so that tests can run using
the test_sh function, which will delay failure until the end of the rake
run, unless the TESTS_FAIL_FAST environment variable is set.

Furthermore, this reorganizes the jasmine test tasks so that we can run
those as part of `rake test` as well.
This commit is contained in:
Calen Pennington
2013-06-05 13:05:00 -04:00
parent fdf213f9b5
commit 0bf7c71ec2
6 changed files with 168 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
doc/testing.md
View File

@@ -141,21 +141,36 @@ Very handy: if you uncomment the `pdb=1` line in `setup.cfg`, it will drop you i
### Running Javascript Unit Tests
These commands start a development server with jasmine testing enabled, and launch your default browser
pointing to those tests
To run all of the javascript unit tests, use
rake browse_jasmine_{lms,cms}
rake jasmine
To run the tests headless, you must install [phantomjs](http://phantomjs.org/download.html), then run:
If the `phantomjs` binary is on the path, or the `PHANTOMJS_PATH` environment variable is
set to point to it, then the tests will be run headless. Otherwise, they will be run in
your default browser
rake phantomjs_jasmine_{lms,cms}
export PATH=/path/to/phantomjs:$PATH
rake jasmine # Runs headless
If the `phantomjs` binary is not on the path, set the `PHANTOMJS_PATH` environment variable to point to it
or
PHANTOMJS_PATH=/path/to/phantomjs rake phantomjs_jasmine_{lms,cms}
PHANTOMJS_PATH=/path/to/phantomjs rake jasmine # Runs headless
Once you have run the `rake` command, your browser should open to
to `http://localhost/_jasmine/`, which displays the test results.
or
rake jasmine # Runs in browser
You can also force a run using phantomjs or the browser using the commands
rake jasmine:browser # Runs in browser
rake jasmine:phantomjs # Runs headless
You can run tests for a specific subsystems as well
rake jasmine:lms # Runs all lms javascript unit tests using the default method
rake jasmine:cms:browser # Runs all cms javascript unit tests in the browser
Use `rake -T` to get a list of all available subsystems
**Troubleshooting**: If you get an error message while running the `rake` task,
try running `bundle install` to install the required ruby gems.

16
jenkins/test.sh
View File

@@ -70,23 +70,11 @@ rake clobber
rake pep8 > pep8.log || cat pep8.log
rake pylint > pylint.log || cat pylint.log
TESTS_FAILED=0
# Run the python unit tests
rake test_cms || TESTS_FAILED=1
rake test_lms || TESTS_FAILED=1
rake test_common/lib/capa || TESTS_FAILED=1
rake test_common/lib/xmodule || TESTS_FAILED=1
# Run the javascript unit tests
rake phantomjs_jasmine_lms || TESTS_FAILED=1
rake phantomjs_jasmine_cms || TESTS_FAILED=1
rake phantomjs_jasmine_common/lib/xmodule || TESTS_FAILED=1
rake phantomjs_jasmine_common/static/coffee || TESTS_FAILED=1
# Run the unit tests (use phantomjs for javascript unit tests)
rake test
rake coverage:xml coverage:html
[ $TESTS_FAILED == '0' ]
rake autodeploy_properties
github_status state:success "passed"

23
rakefiles/deprecated.rake Normal file
View File

@@ -0,0 +1,23 @@
require 'colorize'
def deprecated(deprecated, deprecated_by)
task deprecated do
puts("Task #{deprecated} has been deprecated. Use #{deprecated_by} instead. Waiting 5 seconds...".red)
sleep(5)
Rake::Task[deprecated_by].invoke
end
end
[:lms, :cms].each do |system|
deprecated("browse_jasmine_#{system}", "jasmine:#{system}:browser")
deprecated("phantomjs_jasmine_#{system}", "jasmine:#{system}:phantomjs")
end
Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib|
deprecated("browse_jasmine_#{lib}", "jasmine:#{lib}:browser")
deprecated("phantomjs_jasmine_#{lib}", "jasmine:#{lib}:phantomjs")
end
deprecated("browse_jasmine_discussion", "jasmine:common/static/coffee:browser")
deprecated("phantomjs_jasmine_discussion", "jasmine:common/static/coffee:phantomjs")

34
rakefiles/helpers.rb
View File

@@ -1,8 +1,12 @@
require 'digest/md5'
def find_executable(exec)
path = %x(which #{exec}).strip
$?.exitstatus == 0 ? path : nil
end
def select_executable(*cmds)
cmds.find_all{ |cmd| system("which #{cmd} > /dev/null 2>&1") }[0] || fail("No executables found from #{cmds.join(', ')}")
cmds.find_all{ |cmd| !find_executable(cmd).nil? }[0] || fail("No executables found from #{cmds.join(', ')}")
end
def django_admin(system, env, command, *args)
@@ -85,3 +89,31 @@ def environments(system)
env_file.gsub("#{system}/envs/", '').gsub(/\.py/, '').gsub('/', '.')
end
end
$failed_tests = 0
# Run sh on args. If TESTS_FAIL_FAST is set, then stop on the first shell failure.
# Otherwise, a final task will be added that will fail if any tests have failed
def test_sh(*args)
sh(*args) do |ok, res|
if ok
return
end
if ENV['TESTS_FAIL_FAST']
fail("Test failed!")
else
$failed_tests += 1
end
end
end
# Add a task after all other tasks that fails if any tests have failed
if !ENV['TESTS_FAIL_FAST']
task :fail_tests do
fail("#{$failed_tests} tests failed!") if $failed_tests > 0
end
Rake.application.top_level_tasks << :fail_tests
end

143
rakefiles/jasmine.rake
View File

@@ -3,6 +3,11 @@ require 'erb'
require 'launchy'
require 'net/http'
PHANTOMJS_PATH = find_executable(ENV['PHANTOMJS_PATH'] || 'phantomjs')
PREFERRED_METHOD = PHANTOMJS_PATH.nil? ? 'browser' : 'phantomjs'
if PHANTOMJS_PATH.nil?
puts("phantomjs not found on path. Set $PHANTOMJS_PATH. Using browser for jasmine tests".blue)
end
def django_for_jasmine(system, django_reload)
if !django_reload
@@ -35,18 +40,6 @@ def django_for_jasmine(system, django_reload)
end
def template_jasmine_runner(lib)
case lib
when /common\/lib\/.+/
coffee_files = Dir["#{lib}/**/js/**/*.coffee", "common/static/coffee/src/**/*.coffee"]
when /common\/static\/coffee/
coffee_files = Dir["#{lib}/**/*.coffee"]
else
puts('I do not know how to run jasmine tests for #{lib}')
exit
end
if !coffee_files.empty?
sh("node_modules/.bin/coffee -c #{coffee_files.join(' ')}")
end
phantom_jasmine_path = File.expand_path("node_modules/phantom-jasmine")
jasmine_reporters_path = File.expand_path("node_modules/jasmine-reporters")
common_js_root = File.expand_path("common/static/js")
@@ -54,8 +47,8 @@ def template_jasmine_runner(lib)
# Get arrays of spec and source files, ordered by how deep they are nested below the library
# (and then alphabetically) and expanded from a relative to an absolute path
spec_glob = File.join("#{lib}", "**", "spec", "**", "*.js")
src_glob = File.join("#{lib}", "**", "src", "**", "*.js")
spec_glob = File.join(lib, "**", "spec", "**", "*.js")
src_glob = File.join(lib, "**", "src", "**", "*.js")
js_specs = Dir[spec_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)}
js_source = Dir[src_glob].sort_by {|p| [p.split('/').length, p]} .map {|f| File.expand_path(f)}
@@ -68,74 +61,90 @@ def template_jasmine_runner(lib)
yield File.expand_path(template_output)
end
def run_phantom_js(url)
phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs'
sh("#{phantomjs} node_modules/jasmine-reporters/test/phantomjs-testrunner.js #{url}")
def jasmine_browser(url, wait=10)
# Jitter starting the browser so that the tests don't all try and
# start the browser simultaneously
sleep(rand(3))
sh("python -m webbrowser -t '#{url}'")
sleep(wait)
end
# Open jasmine tests for :system in the default browser. The :env
# should (always?) be 'jasmine', but it's passed as an arg so that
# the :assets dependency gets it.
#
# This task should be invoked via the wrapper below, so we don't
# include a description to keep it from showing up in rake -T.
task :browse_jasmine, [:system, :env] => :assets do |t, args|
django_for_jasmine(args.system, true) do |jasmine_url|
Launchy.open(jasmine_url)
puts "Press ENTER to terminate".red
$stdin.gets
end
end
# Use phantomjs to run jasmine tests from the console. The :env
# should (always?) be 'jasmine', but it's passed as an arg so that
# the :assets dependency gets it.
#
# This task should be invoked via the wrapper below, so we don't
# include a description to keep it from showing up in rake -T.
task :phantomjs_jasmine, [:system, :env] => :assets do |t, args|
django_for_jasmine(args.system, false) do |jasmine_url|
run_phantom_js(jasmine_url)
end
def jasmine_phantomjs(url)
fail("phantomjs not found. Add it to your path, or set $PHANTOMJS_PATH") if PHANTOMJS_PATH.nil?
test_sh("#{PHANTOMJS_PATH} node_modules/jasmine-reporters/test/phantomjs-testrunner.js #{url}")
end
# Wrapper tasks for the real browse_jasmine and phantomjs_jasmine
# tasks above. These have a nicer UI since there's no arg passing.
[:lms, :cms].each do |system|
desc "Open jasmine tests for #{system} in your default browser"
task "browse_jasmine_#{system}" do
Rake::Task[:browse_jasmine].invoke(system, 'jasmine')
end
namespace :jasmine do
namespace system do
desc "Open jasmine tests for #{system} in your default browser"
task :browser do
Rake::Task[:assets].invoke(system, 'jasmine')
django_for_jasmine(system, true) do |jasmine_url|
jasmine_browser(jasmine_url)
end
end
desc "Use phantomjs to run jasmine tests for #{system} from the console"
task "phantomjs_jasmine_#{system}" do
Rake::Task[:phantomjs_jasmine].invoke(system, 'jasmine')
desc "Use phantomjs to run jasmine tests for #{system} from the console"
task :phantomjs do
Rake::Task[:assets].invoke(system, 'jasmine')
phantomjs = ENV['PHANTOMJS_PATH'] || 'phantomjs'
django_for_jasmine(system, false) do |jasmine_url|
jasmine_phantomjs(jasmine_url)
end
end
end
desc "Run jasmine tests for #{system} using #{PREFERRED_METHOD}"
task system => "jasmine:#{system}:#{PREFERRED_METHOD}"
task :phantomjs => "jasmine:#{system}:phantomjs"
multitask :browser => "jasmine:#{system}:browser"
end
end
STATIC_JASMINE_TESTS = Dir["common/lib/*"].select{|lib| File.directory?(lib)}
STATIC_JASMINE_TESTS << 'common/static/coffee'
static_js_dirs = Dir["common/lib/*"].select{|lib| File.directory?(lib)}
static_js_dirs << 'common/static/coffee'
static_js_dirs.select!{|lib| !Dir["#{lib}/**/spec"].empty?}
STATIC_JASMINE_TESTS.each do |lib|
desc "Open jasmine tests for #{lib} in your default browser"
task "browse_jasmine_#{lib}" do
template_jasmine_runner(lib) do |f|
sh("python -m webbrowser -t 'file://#{f}'")
puts "Press ENTER to terminate".red
$stdin.gets
end
end
static_js_dirs.each do |dir|
namespace :jasmine do
namespace dir do
desc "Open jasmine tests for #{dir} in your default browser"
task :browser do
# We need to use either CMS or LMS to preprocess files. Use LMS by default
Rake::Task['assets:coffee'].invoke('lms', 'jasmine')
template_jasmine_runner(dir) do |f|
jasmine_browser("file://#{f}")
end
end
desc "Use phantomjs to run jasmine tests for #{lib} from the console"
task "phantomjs_jasmine_#{lib}" do
template_jasmine_runner(lib) do |f|
run_phantom_js(f)
desc "Use phantomjs to run jasmine tests for #{dir} from the console"
task :phantomjs do
# We need to use either CMS or LMS to preprocess files. Use LMS by default
Rake::Task[:assets].invoke('lms', 'jasmine')
template_jasmine_runner(dir) do |f|
jasmine_phantomjs(f)
end
end
end
desc "Run jasmine tests for #{dir} using #{PREFERRED_METHOD}"
task dir => "jasmine:#{dir}:#{PREFERRED_METHOD}"
task :phantomjs => "jasmine:#{dir}:phantomjs"
multitask :browser => "jasmine:#{dir}:browser"
end
end
desc "Open jasmine tests for discussion in your default browser"
task "browse_jasmine_discussion" => "browse_jasmine_common/static/coffee"
desc "Run all jasmine tests using #{PREFERRED_METHOD}"
task :jasmine => "jasmine:#{PREFERRED_METHOD}"
desc "Use phantomjs to run jasmine tests for discussion from the console"
task "phantomjs_jasmine_discussion" => "phantomjs_jasmine_common/static/coffee"
['phantomjs', 'browser'].each do |method|
desc "Run all jasmine tests using #{method}"
task "jasmine:#{method}"
end
task :test => :jasmine

36
rakefiles/tests.rake
View File

@@ -1,9 +1,6 @@
# Set up the clean and clobber tasks
CLOBBER.include(REPORT_DIR, 'test_root/*_repo', 'test_root/staticfiles')
$failed_tests = 0
def run_under_coverage(cmd, root)
cmd0, cmd_rest = cmd.split(" ", 2)
# We use "python -m coverage" so that the proper python will run the importable coverage
@@ -17,12 +14,7 @@ def run_tests(system, report_dir, test_id=nil, stop_on_failure=true)
dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"]
test_id = dirs.join(' ') if test_id.nil? or test_id == ''
cmd = django_admin(system, :test, 'test', '--logging-clear-handlers', test_id)
sh(run_under_coverage(cmd, system)) do |ok, res|
if !ok and stop_on_failure
abort "Test failed!"
end
$failed_tests += 1 unless ok
end
test_sh(run_under_coverage(cmd, system))
end
def run_acceptance_tests(system, report_dir, harvest_args)
@@ -38,7 +30,7 @@ def run_acceptance_tests(system, report_dir, harvest_args)
end
sh(django_admin(system, 'acceptance', 'syncdb', '--noinput'))
sh(django_admin(system, 'acceptance', 'migrate', '--noinput'))
sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--tag -skip', harvest_args))
test_sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--tag -skip', harvest_args))
end
@@ -55,13 +47,13 @@ TEST_TASK_DIRS = []
# Per System tasks
desc "Run all django tests on our djangoapps for the #{system}"
task "test_#{system}", [:test_id, :stop_on_failure] => ["clean_test_files", :predjango, "#{system}:gather_assets:test", "fasttest_#{system}"]
task "test_#{system}", [:test_id] => ["clean_test_files", :predjango, "#{system}:gather_assets:test", "fasttest_#{system}"]
# Have a way to run the tests without running collectstatic -- useful when debugging without
# messing with static files.
task "fasttest_#{system}", [:test_id, :stop_on_failure] => [report_dir, :install_prereqs, :predjango] do |t, args|
args.with_defaults(:stop_on_failure => 'true', :test_id => nil)
run_tests(system, report_dir, args.test_id, args.stop_on_failure)
task "fasttest_#{system}", [:test_id] => [report_dir, :install_prereqs, :predjango] do |t, args|
args.with_defaults(:test_id => nil)
run_tests(system, report_dir, args.test_id)
end
# Run acceptance tests
@@ -89,9 +81,7 @@ Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib|
task task_name => report_dir do
ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml")
cmd = "nosetests #{lib}"
sh(run_under_coverage(cmd, lib)) do |ok, res|
$failed_tests += 1 unless ok
end
test_sh(run_under_coverage(cmd, lib))
end
TEST_TASK_DIRS << lib
@@ -107,17 +97,11 @@ TEST_TASK_DIRS.each do |dir|
report_dir = report_dir_path(dir)
directory report_dir
task :report_dirs => [REPORT_DIR, report_dir]
task :test => "test_#{dir}"
end
task :test do
TEST_TASK_DIRS.each do |dir|
Rake::Task["test_#{dir}"].invoke(nil, false)
end
if $failed_tests > 0
abort "Tests failed!"
end
end
desc "Run all tests"
task :test
namespace :coverage do
desc "Build the html coverage reports"