Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Write xunit reports for acceptance tests

Browse Source 
Refactor rake tasks for acceptance tests
Address PR comments
This commit is contained in:
Jay Zoldak
2013-09-19 14:40:27 -04:00
parent 761590bc8f
commit 6ebb753823
5 changed files with 84 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
docs/internal/testing.md
View File

@@ -205,24 +205,27 @@ with Chrome (not Chromium) version 28.0.1500.71 with ChromeDriver
version 2.1.210398.
To run all the acceptance tests:
rake test:acceptance
rake test_acceptance_lms
rake test_acceptance_cms
To run only for lms or cms:
rake test:acceptance:lms
rake test:acceptance:cms
To test only a specific feature:
rake test_acceptance_lms["lms/djangoapps/courseware/features/problems.feature"]
rake test:acceptance:lms["lms/djangoapps/courseware/features/problems.feature"]
To test only a specific scenario
rake test_acceptance_lms["lms/djangoapps/courseware/features/problems.feature -s 3"]
rake test:acceptance:lms["lms/djangoapps/courseware/features/problems.feature -s 3"]
To start the debugger on failure, add the `--pdb` option:
rake test_acceptance_lms["lms/djangoapps/courseware/features/problems.feature --pdb"]
rake test:acceptance:lms["lms/djangoapps/courseware/features/problems.feature --pdb"]
To run tests faster by not collecting static files, you can use
`rake fasttest_acceptance_lms` and `rake fasttest_acceptance_cms`.
`rake test:acceptance:lms:fast` and `rake test:acceptance:cms:fast`.
Acceptance tests will run on a randomized port and can be run in the background of rake cms and lms or unit tests.
To specify the port, change the LETTUCE_SERVER_PORT constant in cms/envs/acceptance.py and lms/envs/acceptance.py

4
jenkins/test_acceptance.sh
View File

@@ -44,8 +44,6 @@ if [ "$LETTUCE_SELENIUM_CLIENT" == saucelabs ]; then
fi
# Run the lms and cms acceptance tests
# (the -v flag turns off color in the output)
rake test_acceptance_lms["-v 3 $SKIP_TESTS"] || TESTS_FAILED=1
rake test_acceptance_cms["-v 3 $SKIP_TESTS"] || TESTS_FAILED=1
rake test:acceptance["$SKIP_TESTS"] || TESTS_FAILED=1
[ $TESTS_FAILED == '0' ]

71
rakelib/acceptance_test.rake Normal file
View File

@@ -0,0 +1,71 @@
ACCEPTANCE_DB = 'test_root/db/test_edx.db'
ACCEPTANCE_REPORT_DIR = report_dir_path('acceptance')
directory ACCEPTANCE_REPORT_DIR
def run_acceptance_tests(system, harvest_args)
# Create the acceptance report directory
# because if it doesn't exist then lettuce will give an IOError.
report_dir = report_dir_path('acceptance')
report_file = File.join(ACCEPTANCE_REPORT_DIR, "#{system}.xml")
report_args = "--with-xunit --xunit-file #{report_file}"
test_sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--verbosity 2', '--tag -skip', report_args, harvest_args))
end
task :setup_acceptance_db do
# HACK: Since the CMS depends on the existence of some database tables
# that are now in common but used to be in LMS (Role/Permissions for Forums)
# we need to create/migrate the database tables defined in the LMS.
# We might be able to address this by moving out the migrations from
# lms/django_comment_client, but then we'd have to repair all the existing
# migrations from the upgrade tables in the DB.
# But for now for either system (lms or cms), use the lms
# definitions to sync and migrate.
if File.exists?(ACCEPTANCE_DB)
File.delete(ACCEPTANCE_DB)
end
sh(django_admin('lms', 'acceptance', 'syncdb', '--noinput'))
sh(django_admin('lms', 'acceptance', 'migrate', '--noinput'))
end
task :prep_for_acceptance_tests => [
:clean_reports_dir, :clean_test_files, ACCEPTANCE_REPORT_DIR,
:install_prereqs, :setup_acceptance_db
]
namespace :test do
namespace :acceptance do
task :all, [:harvest_args] => [
:prep_for_acceptance_tests,
"^^lms:gather_assets:acceptance",
"^^cms:gather_assets:acceptance"
] do |t, args|
run_acceptance_tests('lms', args.harvest_args)
run_acceptance_tests('cms', args.harvest_args)
end
['lms', 'cms'].each do |system|
desc "Run the acceptance tests for the #{system}"
task system, [:harvest_args] => [
:prep_for_acceptance_tests,
"^^#{system}:gather_assets:acceptance"
] do |t, args|
args.with_defaults(:harvest_args => '')
run_acceptance_tests(system, args.harvest_args)
end
desc "Run acceptance tests for the #{system} without collectstatic or db migrations"
task "#{system}:fast", [:harvest_args] => [
:clean_reports_dir, ACCEPTANCE_REPORT_DIR,
] do |t, args|
args.with_defaults(:harvest_args => '')
run_acceptance_tests(system, args.harvest_args)
end
end
end
desc "Run the lettuce acceptance tests for lms and cms"
task :acceptance, [:harvest_args] do |t, args|
Rake::Task["test:acceptance:all"].invoke(args.harvest_args)
end
end

3
rakelib/deprecated.rake
View File

@@ -24,6 +24,8 @@ end
deprecated("jasmine:#{system}:phantomjs", "test:js:run", system)
deprecated("#{system}:check_settings:jasmine", "")
deprecated("#{system}:gather_assets:jasmine", "")
deprecated("test_acceptance_#{system}", "test:acceptance:#{system}")
deprecated("fasttest_acceptance_#{system}", "test:acceptance:#{system}:fast")
end
Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib|
@@ -49,3 +51,4 @@ deprecated("jasmine:common/static/coffee:phantomjs", "test:js:run", "common")
deprecated("jasmine", "test:js")
deprecated("jasmine:phantomjs", "test:js:run")
deprecated("jasmine:browser", "test:js:dev")
deprecated("test_acceptance", "test:acceptance")

54
rakelib/tests.rake
View File

@@ -4,8 +4,6 @@ CLOBBER.include(REPORT_DIR, 'test_root/*_repo', 'test_root/staticfiles')
# Create the directory to hold coverage reports, if it doesn't already exist.
directory REPORT_DIR
ACCEPTANCE_DB = 'test_root/db/test_edx.db'
def test_id_dir(path)
return File.join(".testids", path.to_s)
end
@@ -32,38 +30,6 @@ def run_tests(system, report_dir, test_id=nil, stop_on_failure=true)
test_sh(run_under_coverage(cmd, system))
end
def create_acceptance_db(system)
# HACK: Since now the CMS depends on the existence of some database tables
# that used to be in LMS (Role/Permissions for Forums) we need to make
# sure the acceptance tests create/migrate the database tables
# that are represented in the LMS. We might be able to address this by moving
# out the migrations from lms/django_comment_client, but then we'd have to
# repair all the existing migrations from the upgrade tables in the DB.
if system == :cms
sh(django_admin('lms', 'acceptance', 'syncdb', '--noinput'))
sh(django_admin('lms', 'acceptance', 'migrate', '--noinput'))
end
sh(django_admin(system, 'acceptance', 'syncdb', '--noinput'))
sh(django_admin(system, 'acceptance', 'migrate', '--noinput'))
end
def setup_acceptance_db(system, fasttest=false)
# If running under fasttest mode and the database already
# exists, skip the migrations.
if File.exists?(ACCEPTANCE_DB)
if not fasttest
File.delete(ACCEPTANCE_DB)
create_acceptance_db(system)
end
else
create_acceptance_db(system)
end
end
def run_acceptance_tests(system, report_dir, harvest_args)
test_sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--verbosity 2', '--tag -skip', harvest_args))
end
# Run documentation tests
desc "Run documentation tests"
task :test_docs do
@@ -110,26 +76,6 @@ TEST_TASK_DIRS = []
run_tests(system, report_dir, args.test_id)
end
# Run acceptance tests
desc "Run acceptance tests"
task "test_acceptance_#{system}", [:harvest_args] => [
:clean_test_files, :install_prereqs,
"#{system}:gather_assets:acceptance"
] do |t, args|
setup_acceptance_db(system)
Rake::Task["fasttest_acceptance_#{system}"].invoke(args.harvest_args)
end
desc "Run acceptance tests without collectstatic or database migrations"
task "fasttest_acceptance_#{system}", [:harvest_args] => [
report_dir, :clean_reports_dir
] do |t, args|
args.with_defaults(:harvest_args => '')
setup_acceptance_db(system, fasttest=true)
run_acceptance_tests(system, report_dir, args.harvest_args)
end
task :fasttest => "fasttest_#{system}"
TEST_TASK_DIRS << system