diff --git a/.gitignore b/.gitignore index 4fd90cfe03..72de96e0c4 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,4 @@ autodeploy.properties .ws_migrations_complete .vagrant/ logs +.testids/ diff --git a/docs/internal/testing.md b/docs/internal/testing.md index ae24e5dbbe..88ef52e16d 100644 --- a/docs/internal/testing.md +++ b/docs/internal/testing.md @@ -103,6 +103,10 @@ You can run tests using `rake` commands. For example, runs all the tests. It also runs `collectstatic`, which prepares the static files used by the site (for example, compiling Coffeescript to Javascript). +You can re-run all failed python tests by running (all JS tests will still run) + + rake test[--failed] + You can also run the tests without `collectstatic`, which tends to be faster: rake fasttest_lms @@ -128,6 +132,10 @@ To run a single django test: rake test_lms[courseware.tests.tests:TestViewAuth.test_dark_launch] +To re-run all failing django tests from lms or cms: + + rake test_lms[--failed] + To run a single nose test file: nosetests common/lib/xmodule/xmodule/tests/test_stringify.py @@ -142,18 +150,18 @@ To run a single test and get stdout, with proper env config: To run a single test and get stdout and get coverage: - python -m coverage run --rcfile=./common/lib/xmodule/.coveragerc which ./manage.py cms --settings test test --traceback --logging-clear-handlers --liveserver=localhost:8000-9000 contentstore.tests.test_import_nostatic -s # cms example + python -m coverage run --rcfile=./common/lib/xmodule/.coveragerc which ./manage.py cms --settings test test --traceback --logging-clear-handlers --liveserver=localhost:8000-9000 contentstore.tests.test_import_nostatic -s # cms example python -m coverage run --rcfile=./lms/.coveragerc which ./manage.py lms --settings test test --traceback --logging-clear-handlers --liveserver=localhost:8000-9000 courseware.tests.test_module_render -s # lms example generate coverage report: - coverage report --rcfile=./common/lib/xmodule/.coveragerc + coverage report --rcfile=./common/lib/xmodule/.coveragerc or to get html report: coverage html --rcfile=./common/lib/xmodule/.coveragerc - -then browse reports/common/lib/xmodule/cover/index.html + +then browse reports/common/lib/xmodule/cover/index.html Very handy: if you uncomment the `pdb=1` line in `setup.cfg`, it will drop you into pdb on error. This lets you go up and down the stack and see what the values of the variables are. Check out [the pdb documentation](http://docs.python.org/library/pdb.html) diff --git a/rakelib/tests.rake b/rakelib/tests.rake index fe5d0a351d..bed5d5b36a 100644 --- a/rakelib/tests.rake +++ b/rakelib/tests.rake @@ -4,6 +4,10 @@ CLOBBER.include(REPORT_DIR, 'test_root/*_repo', 'test_root/staticfiles') # Create the directory to hold coverage reports, if it doesn't already exist. directory REPORT_DIR +def test_id_dir(path) + return File.join(".testids", path.to_s) +end + def run_under_coverage(cmd, root) cmd0, cmd_rest = cmd.split(" ", 2) # We use "python -m coverage" so that the proper python will run the importable coverage @@ -14,9 +18,15 @@ end def run_tests(system, report_dir, test_id=nil, stop_on_failure=true) ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml") + test_id_file = File.join(test_id_dir(system), "noseids") dirs = Dir["common/djangoapps/*"] + Dir["#{system}/djangoapps/*"] test_id = dirs.join(' ') if test_id.nil? or test_id == '' - cmd = django_admin(system, :test, 'test', '--logging-clear-handlers', '--liveserver=localhost:8000-9000', test_id) + cmd = django_admin( + system, :test, 'test', + '--logging-clear-handlers', + '--liveserver=localhost:8000-9000', + "--id-file=#{test_id_file}", + test_id) test_sh(run_under_coverage(cmd, system)) end @@ -64,14 +74,17 @@ TEST_TASK_DIRS = [] [:lms, :cms].each do |system| report_dir = report_dir_path(system) + test_id_dir = test_id_dir(system) - # Per System tasks + directory test_id_dir + + # Per System tasks/ desc "Run all django tests on our djangoapps for the #{system}" task "test_#{system}", [:test_id] => [:clean_test_files, :predjango, "#{system}:gather_assets:test", "fasttest_#{system}"] # Have a way to run the tests without running collectstatic -- useful when debugging without # messing with static files. - task "fasttest_#{system}", [:test_id] => [report_dir, :clean_reports_dir, :install_prereqs, :predjango] do |t, args| + task "fasttest_#{system}", [:test_id] => [test_id_dir, report_dir, :clean_reports_dir, :install_prereqs, :predjango] do |t, args| args.with_defaults(:test_id => nil) run_tests(system, report_dir, args.test_id) end @@ -97,12 +110,16 @@ end Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib| report_dir = report_dir_path(lib) + test_id_dir = test_id_dir(lib) + test_ids = File.join(test_id_dir(lib), '.noseids') + + directory test_id_dir desc "Run tests for common lib #{lib}" - task "test_#{lib}", [:test_id] => [report_dir, :clean_reports_dir] do |t, args| + task "test_#{lib}", [:test_id] => [test_id_dir, report_dir, :clean_reports_dir] do |t, args| args.with_defaults(:test_id => lib) ENV['NOSE_XUNIT_FILE'] = File.join(report_dir, "nosetests.xml") - cmd = "nosetests #{args.test_id}" + cmd = "nosetests --id-file=#{test_ids} #{args.test_id}" test_sh(run_under_coverage(cmd, lib)) end TEST_TASK_DIRS << lib @@ -126,7 +143,7 @@ TEST_TASK_DIRS.each do |dir| end desc "Run all tests" -task :test => :test_docs +task :test, [:test_id] => :test_docs desc "Build the html, xml, and diff coverage reports" task :coverage => :report_dirs do diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index d700aaa195..7e65eda713 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -89,5 +89,6 @@ django-jasmine==0.3.2 django_debug_toolbar django-debug-toolbar-mongo nose-ignore-docstring +nose-exclude git+https://github.com/mfogel/django-settings-context-processor.git diff --git a/setup.cfg b/setup.cfg index 844df75ce8..da9525a300 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,6 +3,9 @@ logging-clear-handlers=1 with-xunit=1 rednose=1 with-ignore-docstrings=1 +with-id=1 +exclude-dir=lms/envs + cms/envs # Uncomment the following line to open pdb when a test fails #pdb=1