diff --git a/cms/djangoapps/contentstore/features/advanced-settings.feature b/cms/djangoapps/contentstore/features/advanced-settings.feature index a11a6cb869..b2941ac7a5 100644 --- a/cms/djangoapps/contentstore/features/advanced-settings.feature +++ b/cms/djangoapps/contentstore/features/advanced-settings.feature @@ -2,6 +2,7 @@ Feature: Advanced (manual) course policy In order to specify course policy settings for which no custom user interface exists I want to be able to manually enter JSON key /value pairs + Scenario: A course author sees default advanced settings Given I have opened a new course in Studio When I select the Advanced Settings @@ -11,6 +12,8 @@ Feature: Advanced (manual) course policy Given I am on the Advanced Course Settings page in Studio Then the settings are alphabetized + # Sauce labs does not play nicely with CodeMirror + @skip_sauce Scenario: Test cancel editing key value Given I am on the Advanced Course Settings page in Studio When I edit the value of a policy key @@ -19,6 +22,8 @@ Feature: Advanced (manual) course policy And I reload the page Then the policy key value is unchanged + # Sauce labs does not play nicely with CodeMirror + @skip_sauce Scenario: Test editing key value Given I am on the Advanced Course Settings page in Studio When I edit the value of a policy key and save @@ -26,6 +31,8 @@ Feature: Advanced (manual) course policy And I reload the page Then the policy key value is changed + # Sauce labs does not play nicely with CodeMirror + @skip_sauce Scenario: Test how multi-line input appears Given I am on the Advanced Course Settings page in Studio When I create a JSON object as a value for "discussion_topics" @@ -33,6 +40,8 @@ Feature: Advanced (manual) course policy And I reload the page Then it is displayed as formatted + # Sauce labs does not play nicely with CodeMirror + @skip_sauce Scenario: Test error if value supplied is of the wrong type Given I am on the Advanced Course Settings page in Studio When I create a JSON object as a value for "display_name" @@ -41,6 +50,8 @@ Feature: Advanced (manual) course policy Then the policy key value is unchanged # This feature will work in Firefox only when Firefox is the active window + # Sauce labs does not play nicely with CodeMirror + @skip_sauce Scenario: Test automatic quoting of non-JSON values Given I am on the Advanced Course Settings page in Studio When I create a non-JSON value not in quotes @@ -48,6 +59,8 @@ Feature: Advanced (manual) course policy And I reload the page Then it is displayed as a string + # Sauce labs does not play nicely with CodeMirror + @skip_sauce Scenario: Confirmation is shown on save Given I am on the Advanced Course Settings page in Studio When I edit the value of a policy key diff --git a/cms/djangoapps/contentstore/features/checklists.feature b/cms/djangoapps/contentstore/features/checklists.feature index f13ce53fc2..6289df9cfc 100644 --- a/cms/djangoapps/contentstore/features/checklists.feature +++ b/cms/djangoapps/contentstore/features/checklists.feature @@ -10,7 +10,10 @@ Feature: Course checklists Then I can check and uncheck tasks in a checklist And They are correctly selected after reloading the page - # CHROME ONLY, due to issues getting link to be active in firefox + # There are issues getting link to be active in browsers other than chrome + @skip_firefox + @skip_internetexplorer + @skip_safari Scenario: A task can link to a location within Studio Given I have opened Checklists When I select a link to the course outline @@ -18,7 +21,10 @@ Feature: Course checklists And I press the browser back button Then I am brought back to the course outline in the correct state - # CHROME ONLY, due to issues getting link to be active in firefox + # There are issues getting link to be active in browsers other than chrome + @skip_firefox + @skip_internetexplorer + @skip_safari Scenario: A task can link to a location outside Studio Given I have opened Checklists When I select a link to help page diff --git a/cms/djangoapps/contentstore/features/course-overview.feature b/cms/djangoapps/contentstore/features/course-overview.feature index a9aed5d982..2cbb22ddd7 100644 --- a/cms/djangoapps/contentstore/features/course-overview.feature +++ b/cms/djangoapps/contentstore/features/course-overview.feature @@ -64,6 +64,10 @@ Feature: Course Overview And I change an assignment's grading status Then I am shown a notification + # Notification is not shown on reorder for IE + # Safari does not have moveMouseTo implemented + @skip_internetexplorer + @skip_safari Scenario: Notification is shown on subsection reorder Given I have opened a new course section in Studio And I have added a new subsection diff --git a/cms/djangoapps/contentstore/features/course-settings.feature b/cms/djangoapps/contentstore/features/course-settings.feature index 8f00452efe..9976179b68 100644 --- a/cms/djangoapps/contentstore/features/course-settings.feature +++ b/cms/djangoapps/contentstore/features/course-settings.feature @@ -1,6 +1,8 @@ Feature: Course Settings As a course author, I want to be able to configure my course settings. + # Safari has trouble keeps dates on refresh + @skip_safari Scenario: User can set course dates Given I have opened a new course in Studio When I select Schedule and Details @@ -8,12 +10,16 @@ Feature: Course Settings And I press the "Save" notification button Then I see the set dates on refresh + # IE has trouble with saving information + @skip_internetexplorer Scenario: User can clear previously set course dates (except start date) Given I have set course dates And I clear all the dates except start And I press the "Save" notification button Then I see cleared dates on refresh + # IE has trouble with saving information + @skip_internetexplorer Scenario: User cannot clear the course start date Given I have set course dates And I press the "Save" notification button @@ -21,6 +27,10 @@ Feature: Course Settings Then I receive a warning about course start date And The previously set start date is shown on refresh + # IE has trouble with saving information + # Safari gets CSRF token errors + @skip_internetexplorer + @skip_safari Scenario: User can correct the course start date warning Given I have tried to clear the course start And I have entered a new course start date @@ -28,12 +38,16 @@ Feature: Course Settings Then The warning about course start date goes away And My new course start date is shown on refresh + # Safari does not save + refresh properly through sauce labs + @skip_safari Scenario: Settings are only persisted when saved Given I have set course dates And I press the "Save" notification button When I change fields Then I do not see the new changes persisted on refresh + # Safari does not save + refresh properly through sauce labs + @skip_safari Scenario: Settings are reset on cancel Given I have set course dates And I press the "Save" notification button @@ -41,6 +55,8 @@ Feature: Course Settings And I press the "Cancel" notification button Then I do not see the changes + # Safari gets CSRF token errors + @skip_safari Scenario: Confirmation is shown on save Given I have opened a new course in Studio When I select Schedule and Details diff --git a/cms/djangoapps/contentstore/features/course-team.py b/cms/djangoapps/contentstore/features/course-team.py index db7b4d81f9..8b31d325e5 100644 --- a/cms/djangoapps/contentstore/features/course-team.py +++ b/cms/djangoapps/contentstore/features/course-team.py @@ -91,7 +91,7 @@ def remove_course_team_admin(_step, outer_capture, name): @step(u'"([^"]*)" logs in$') def other_user_login(_step, name): - world.browser.cookies.delete() + world.visit('logout') world.visit('/') signin_css = 'a.action-signin' diff --git a/cms/djangoapps/contentstore/features/course-updates.feature b/cms/djangoapps/contentstore/features/course-updates.feature index fb18e51f2d..41ee785db5 100644 --- a/cms/djangoapps/contentstore/features/course-updates.feature +++ b/cms/djangoapps/contentstore/features/course-updates.feature @@ -1,6 +1,8 @@ Feature: Course updates As a course author, I want to be able to provide updates to my students + # Internet explorer can't select all so the update appears weirdly + @skip_internetexplorer Scenario: Users can add updates Given I have opened a new course in Studio And I go to the course updates page @@ -8,6 +10,8 @@ Feature: Course updates Then I should see the update "Hello" And I see a "saving" notification + # Internet explorer can't select all so the update appears weirdly + @skip_internetexplorer Scenario: Users can edit updates Given I have opened a new course in Studio And I go to the course updates page @@ -33,6 +37,8 @@ Feature: Course updates Then I should see the date "June 1, 2013" And I see a "saving" notification + # Internet explorer can't select all so the update appears weirdly + @skip_internetexplorer Scenario: Users can change handouts Given I have opened a new course in Studio And I go to the course updates page diff --git a/cms/djangoapps/contentstore/features/discussion-editor.feature b/cms/djangoapps/contentstore/features/discussion-editor.feature index 8fb14c3205..e4b1f5450b 100644 --- a/cms/djangoapps/contentstore/features/discussion-editor.feature +++ b/cms/djangoapps/contentstore/features/discussion-editor.feature @@ -6,6 +6,8 @@ Feature: Discussion Component Editor And I edit and select Settings Then I see three alphabetized settings and their expected values + # Safari doesn't save the name properly + @skip_safari Scenario: User can modify display name Given I have created a Discussion Tag And I edit and select Settings diff --git a/cms/djangoapps/contentstore/features/grading.feature b/cms/djangoapps/contentstore/features/grading.feature index 5cf5a72866..0b34feb7aa 100644 --- a/cms/djangoapps/contentstore/features/grading.feature +++ b/cms/djangoapps/contentstore/features/grading.feature @@ -13,7 +13,7 @@ Feature: Course Grading When I add "6" new grades Then I see I now have "5" grades - #Cannot reliably make the delete button appear so using javascript instead + # Cannot reliably make the delete button appear so using javascript instead Scenario: Users can delete grading ranges Given I have opened a new course in Studio And I am viewing the grading settings @@ -21,6 +21,9 @@ Feature: Course Grading And I delete a grade Then I see I now have "2" grades + # IE and Safari cannot reliably drag and drop through selenium + @skip_internetexplorer + @skip_safari Scenario: Users can move grading ranges Given I have opened a new course in Studio And I am viewing the grading settings @@ -85,6 +88,9 @@ Feature: Course Grading When I change assignment type "Homework" to "" Then the save button is disabled + # IE and Safari cannot type in grade range name + @skip_internetexplorer + @skip_safari Scenario: User can edit grading range names Given I have opened a new course in Studio And I have populated the course diff --git a/cms/djangoapps/contentstore/features/grading.py b/cms/djangoapps/contentstore/features/grading.py index 719b3f7f7c..93e44b3893 100644 --- a/cms/djangoapps/contentstore/features/grading.py +++ b/cms/djangoapps/contentstore/features/grading.py @@ -112,10 +112,10 @@ def changes_not_persisted(step): @step(u'I see the assignment type "(.*)"$') def i_see_the_assignment_type(_step, name): - assignment_css = '#course-grading-assignment-name' - assignments = world.css_find(assignment_css) - types = [ele['value'] for ele in assignments] - assert name in types + assignment_css = '#course-grading-assignment-name' + assignments = world.css_find(assignment_css) + types = [ele['value'] for ele in assignments] + assert name in types @step(u'I change the highest grade range to "(.*)"$') @@ -144,6 +144,7 @@ def cannot_edit_fail(_step): pass # We should get this exception on failing to edit the element + @step(u'I change the grace period to "(.*)"$') def i_change_grace_period(_step, grace_period): grace_period_css = '#course-grading-graceperiod' diff --git a/cms/djangoapps/contentstore/features/html-editor.feature b/cms/djangoapps/contentstore/features/html-editor.feature index 4cd5e1c1b9..4419d6018b 100644 --- a/cms/djangoapps/contentstore/features/html-editor.feature +++ b/cms/djangoapps/contentstore/features/html-editor.feature @@ -6,6 +6,8 @@ Feature: HTML Editor And I edit and select Settings Then I see only the HTML display name setting + # Safari doesn't save the name properly + @skip_safari Scenario: User can modify display name Given I have created a Blank HTML Page And I edit and select Settings diff --git a/cms/djangoapps/contentstore/features/problem-editor.feature b/cms/djangoapps/contentstore/features/problem-editor.feature index 50c49a1896..1296acec1c 100644 --- a/cms/djangoapps/contentstore/features/problem-editor.feature +++ b/cms/djangoapps/contentstore/features/problem-editor.feature @@ -7,12 +7,16 @@ Feature: Problem Editor Then I see five alphabetized settings and their expected values And Edit High Level Source is not visible + # Safari is having trouble saving the values on sauce + @skip_safari Scenario: User can modify String values Given I have created a Blank Common Problem When I edit and select Settings Then I can modify the display name And my display name change is persisted on save + # Safari is having trouble saving the values on sauce + @skip_safari Scenario: User can specify special characters in String values Given I have created a Blank Common Problem When I edit and select Settings @@ -25,6 +29,8 @@ Feature: Problem Editor Then I can revert the display name to unset And my display name is unset on save + # IE will not click the revert button properly + @skip_internetexplorer Scenario: User can select values in a Select Given I have created a Blank Common Problem When I edit and select Settings @@ -32,6 +38,8 @@ Feature: Problem Editor And my change to randomization is persisted And I can revert to the default value for randomization + # Safari will input it as 35. + @skip_safari Scenario: User can modify float input values Given I have created a Blank Common Problem When I edit and select Settings @@ -44,16 +52,22 @@ Feature: Problem Editor When I edit and select Settings Then if I set the weight to "abc", it remains unset + # Safari will input it as 234. + @skip_safari Scenario: User cannot type decimal values integer number field Given I have created a Blank Common Problem When I edit and select Settings Then if I set the max attempts to "2.34", it will persist as a valid integer + # Safari will input it incorrectly + @skip_safari Scenario: User cannot type out of range values in an integer number field Given I have created a Blank Common Problem When I edit and select Settings Then if I set the max attempts to "-3", it will persist as a valid integer + # Safari will input it as 35. + @skip_safari Scenario: Settings changes are not saved on Cancel Given I have created a Blank Common Problem When I edit and select Settings @@ -67,6 +81,8 @@ Feature: Problem Editor Then Edit High Level Source is visible # This feature will work in Firefox only when Firefox is the active window + # IE will not interact with the high level source in sauce labs + @skip_internetexplorer Scenario: High Level source is persisted for LaTeX problem (bug STUD-280) Given I have created a LaTeX Problem When I edit and compile the High Level Source diff --git a/cms/djangoapps/contentstore/features/static-pages.feature b/cms/djangoapps/contentstore/features/static-pages.feature index 9997df69f0..c1a8ec91fc 100644 --- a/cms/djangoapps/contentstore/features/static-pages.feature +++ b/cms/djangoapps/contentstore/features/static-pages.feature @@ -15,6 +15,8 @@ Feature: Static Pages And I "delete" the "Empty" page Then I should not see a "Empty" static page + # Safari won't update the name properly + @skip_safari Scenario: Users can edit static pages Given I have opened a new course in Studio And I go to the static pages page diff --git a/cms/djangoapps/contentstore/features/subsection.feature b/cms/djangoapps/contentstore/features/subsection.feature index 84755b3644..6703c60c3b 100644 --- a/cms/djangoapps/contentstore/features/subsection.feature +++ b/cms/djangoapps/contentstore/features/subsection.feature @@ -25,6 +25,8 @@ Feature: Create Subsection And I reload the page Then I see it marked as Homework + # Safari has trouble saving the date in Sauce + @skip_safari Scenario: Set a due date in a different year (bug #256) Given I have opened a new subsection in Studio And I set the subsection release date to 12/25/2011 03:00 diff --git a/cms/djangoapps/contentstore/features/textbooks.feature b/cms/djangoapps/contentstore/features/textbooks.feature index 0758a0b57b..36de10daa1 100644 --- a/cms/djangoapps/contentstore/features/textbooks.feature +++ b/cms/djangoapps/contentstore/features/textbooks.feature @@ -5,6 +5,9 @@ Feature: Textbooks When I go to the textbooks page Then I should see a message telling me to create a new textbook + # IE and Safari on sauce labs will not upload the textbook correctly resulting in an error + @skip_internetexplorer + @skip_safari Scenario: Create a textbook Given I have opened a new course in Studio And I go to the textbooks page diff --git a/cms/djangoapps/contentstore/features/upload.feature b/cms/djangoapps/contentstore/features/upload.feature index 8d40163685..441de597ea 100644 --- a/cms/djangoapps/contentstore/features/upload.feature +++ b/cms/djangoapps/contentstore/features/upload.feature @@ -1,6 +1,8 @@ Feature: Upload Files As a course author, I want to be able to upload files for my students + # Uploading isn't working on safari with sauce labs + @skip_safari Scenario: Users can upload files Given I have opened a new course in Studio And I go to the files and uploads page @@ -8,6 +10,8 @@ Feature: Upload Files Then I should see the file "test" was uploaded And The url for the file "test" is valid + # Uploading isn't working on safari with sauce labs + @skip_safari Scenario: Users can update files Given I have opened a new course in studio And I go to the files and uploads page @@ -15,6 +19,8 @@ Feature: Upload Files And I upload the file "test" Then I should see only one "test" + # Uploading isn't working on safari with sauce labs + @skip_safari Scenario: Users can delete uploaded files Given I have opened a new course in studio And I go to the files and uploads page @@ -23,12 +29,16 @@ Feature: Upload Files Then I should not see the file "test" was uploaded And I see a confirmation that the file was deleted + # Uploading isn't working on safari with sauce labs + @skip_safari Scenario: Users can download files Given I have opened a new course in studio And I go to the files and uploads page When I upload the file "test" Then I can download the correct "test" file + # Uploading isn't working on safari with sauce labs + @skip_safari Scenario: Users can download updated files Given I have opened a new course in studio And I go to the files and uploads page diff --git a/cms/djangoapps/contentstore/features/upload.py b/cms/djangoapps/contentstore/features/upload.py index a989d6c07f..882b36e6b2 100644 --- a/cms/djangoapps/contentstore/features/upload.py +++ b/cms/djangoapps/contentstore/features/upload.py @@ -10,6 +10,7 @@ import os TEST_ROOT = settings.COMMON_TEST_DATA_ROOT + @step(u'I go to the files and uploads page') def go_to_uploads(_step): menu_css = 'li.nav-course-courseware' @@ -106,8 +107,8 @@ def get_index(file_name): def get_file(file_name): index = get_index(file_name) assert index != -1 - url_css = 'a.filename' + def get_url(): return world.css_find(url_css)[index]._element.get_attribute('href') url = world.retry_on_exception(get_url) diff --git a/cms/djangoapps/contentstore/features/video-editor.feature b/cms/djangoapps/contentstore/features/video-editor.feature index a53183e37c..d238a7e523 100644 --- a/cms/djangoapps/contentstore/features/video-editor.feature +++ b/cms/djangoapps/contentstore/features/video-editor.feature @@ -6,17 +6,23 @@ Feature: Video Component Editor And I edit the component Then I see the correct video settings and default values + # Safari has trouble saving values on Sauce + @skip_safari Scenario: User can modify Video display name Given I have created a Video component And I edit the component Then I can modify the display name And my video display name change is persisted on save + # Sauce Labs cannot delete cookies + @skip_sauce Scenario: Captions are hidden when "show captions" is false Given I have created a Video component And I have set "show captions" to False Then when I view the video it does not show the captions + # Sauce Labs cannot delete cookies + @skip_sauce Scenario: Captions are shown when "show captions" is true Given I have created a Video component And I have set "show captions" to True diff --git a/cms/djangoapps/contentstore/features/video.feature b/cms/djangoapps/contentstore/features/video.feature index 50c06fde63..d2f9915f55 100644 --- a/cms/djangoapps/contentstore/features/video.feature +++ b/cms/djangoapps/contentstore/features/video.feature @@ -10,15 +10,21 @@ Feature: Video Component Given I have clicked the new unit button Then creating a video takes a single click + # Sauce Labs cannot delete cookies + @skip_sauce Scenario: Captions are hidden correctly Given I have created a Video component And I have hidden captions Then when I view the video it does not show the captions + # Sauce Labs cannot delete cookies + @skip_sauce Scenario: Captions are shown correctly Given I have created a Video component Then when I view the video it does show the captions + # Sauce Labs cannot delete cookies + @skip_sauce Scenario: Captions are toggled correctly Given I have created a Video component And I have toggled captions diff --git a/cms/envs/acceptance.py b/cms/envs/acceptance.py index 7debfe18d1..3b89e2e988 100644 --- a/cms/envs/acceptance.py +++ b/cms/envs/acceptance.py @@ -8,6 +8,7 @@ so that we can run the lettuce acceptance tests. # pylint: disable=W0401, W0614 from .test import * +from lms.envs.sauce import * # You need to start the server in debug mode, # otherwise the browser will not render the pages correctly @@ -17,7 +18,7 @@ DEBUG = True import logging logging.disable(logging.ERROR) import os -import random +from random import choice, randint def seed(): @@ -75,7 +76,6 @@ DATABASES = { # Use the auto_auth workflow for creating users and logging them in MITX_FEATURES['AUTOMATIC_AUTH_FOR_TESTING'] = True - # HACK # Setting this flag to false causes imports to not load correctly in the lettuce python files # We do not yet understand why this occurs. Setting this to true is a stopgap measure @@ -84,5 +84,5 @@ USE_I18N = True # Include the lettuce app for acceptance testing, including the 'harvest' django-admin command INSTALLED_APPS += ('lettuce.django',) LETTUCE_APPS = ('contentstore',) -LETTUCE_SERVER_PORT = random.randint(1024, 65535) -LETTUCE_BROWSER = 'chrome' +LETTUCE_SERVER_PORT = choice(PORTS) if SAUCE.get('SAUCE_ENABLED') else randint(1024, 65535) +LETTUCE_BROWSER = os.environ.get('LETTUCE_BROWSER', 'chrome') diff --git a/common/djangoapps/terrain/browser.py b/common/djangoapps/terrain/browser.py index c2bf2bbbf3..cf53aa4f69 100644 --- a/common/djangoapps/terrain/browser.py +++ b/common/djangoapps/terrain/browser.py @@ -11,6 +11,10 @@ from logging import getLogger from django.core.management import call_command from django.conf import settings from selenium.common.exceptions import WebDriverException +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +from requests import put +from base64 import encodestring +from json import dumps # Let the LMS and CMS do their one-time setup # For example, setting up mongo caches @@ -42,43 +46,93 @@ LOGGER.info("Loading the lettuce acceptance testing terrain file...") MAX_VALID_BROWSER_ATTEMPTS = 20 +def get_username_and_key(): + """ + Returns the Sauce Labs username and access ID as set by environment variables + """ + return {"username": settings.SAUCE.get('USERNAME'), "access-key": settings.SAUCE.get('ACCESS_ID')} + + +def set_job_status(jobid, passed=True): + """ + Sets the job status on sauce labs + """ + body_content = dumps({"passed": passed}) + config = get_username_and_key() + base64string = encodestring('{}:{}'.format(config['username'], config['access-key']))[:-1] + result = put('http://saucelabs.com/rest/v1/{}/jobs/{}'.format(config['username'], world.jobid), + data=body_content, + headers={"Authorization": "Basic {}".format(base64string)}) + return result.status_code == 200 + + +def make_desired_capabilities(): + """ + Returns a DesiredCapabilities object corresponding to the environment sauce parameters + """ + desired_capabilities = settings.SAUCE.get('BROWSER', DesiredCapabilities.CHROME) + desired_capabilities['platform'] = settings.SAUCE.get('PLATFORM') + desired_capabilities['version'] = settings.SAUCE.get('VERSION') + desired_capabilities['device-type'] = settings.SAUCE.get('DEVICE') + desired_capabilities['name'] = settings.SAUCE.get('SESSION') + desired_capabilities['build'] = settings.SAUCE.get('BUILD') + desired_capabilities['video-upload-on-pass'] = False + desired_capabilities['sauce-advisor'] = False + desired_capabilities['record-screenshots'] = False + desired_capabilities['selenium-version'] = "2.34.0" + desired_capabilities['max-duration'] = 3600 + desired_capabilities['public'] = 'public restricted' + return desired_capabilities + + @before.harvest def initial_setup(server): """ Launch the browser once before executing the tests. """ - browser_driver = getattr(settings, 'LETTUCE_BROWSER', 'chrome') + world.absorb(settings.SAUCE.get('SAUCE_ENABLED'), 'SAUCE_ENABLED') - # There is an issue with ChromeDriver2 r195627 on Ubuntu - # in which we sometimes get an invalid browser session. - # This is a work-around to ensure that we get a valid session. - success = False - num_attempts = 0 - while (not success) and num_attempts < MAX_VALID_BROWSER_ATTEMPTS: + if not world.SAUCE_ENABLED: + browser_driver = getattr(settings, 'LETTUCE_BROWSER', 'chrome') - # Get a browser session - world.browser = Browser(browser_driver) + # There is an issue with ChromeDriver2 r195627 on Ubuntu + # in which we sometimes get an invalid browser session. + # This is a work-around to ensure that we get a valid session. + success = False + num_attempts = 0 + while (not success) and num_attempts < MAX_VALID_BROWSER_ATTEMPTS: + world.browser = Browser(browser_driver) - # Try to visit the main page - # If the browser session is invalid, this will - # raise a WebDriverException - try: - world.visit('/') + # Try to visit the main page + # If the browser session is invalid, this will + # raise a WebDriverException + try: + world.visit('/') - except WebDriverException: - world.browser.quit() - num_attempts += 1 + except WebDriverException: + world.browser.quit() + num_attempts += 1 - else: - success = True + else: + success = True - # If we were unable to get a valid session within the limit of attempts, - # then we cannot run the tests. - if not success: - raise IOError("Could not acquire valid {driver} browser session.".format(driver=browser_driver)) + # If we were unable to get a valid session within the limit of attempts, + # then we cannot run the tests. + if not success: + raise IOError("Could not acquire valid {driver} browser session.".format(driver=browser_driver)) - # Set the browser size to 1280x1024 - world.browser.driver.set_window_size(1280, 1024) + world.browser.driver.set_window_size(1280, 1024) + + else: + config = get_username_and_key() + world.browser = Browser( + 'remote', + url="http://{}:{}@ondemand.saucelabs.com:80/wd/hub".format(config['username'], config['access-key']), + **make_desired_capabilities() + ) + world.browser.driver.implicitly_wait(30) + + world.absorb(world.browser.driver.session_id, 'jobid') @before.each_scenario @@ -97,7 +151,6 @@ def clear_data(scenario): world.spew('scenario_dict') - @after.each_scenario def reset_databases(scenario): ''' @@ -128,4 +181,6 @@ def teardown_browser(total): """ Quit the browser after executing the tests. """ + if world.SAUCE_ENABLED: + set_job_status(world.jobid, total.scenarios_ran == total.scenarios_passed) world.browser.quit() diff --git a/common/djangoapps/terrain/steps.py b/common/djangoapps/terrain/steps.py index 9cf2aeda49..f13b3ff932 100644 --- a/common/djangoapps/terrain/steps.py +++ b/common/djangoapps/terrain/steps.py @@ -99,7 +99,7 @@ def i_am_logged_in_user(step): @step('I am not logged in$') def i_am_not_logged_in(step): - world.browser.cookies.delete() + world.visit('logout') @step('I am staff for course "([^"]*)"$') @@ -138,10 +138,13 @@ def should_have_link_with_path_and_text(step, path, text): @step(r'should( not)? see "(.*)" (?:somewhere|anywhere) (?:in|on) (?:the|this) page') def should_see_in_the_page(step, doesnt_appear, text): + multiplier = 1 + if world.SAUCE_ENABLED: + multiplier = 2 if doesnt_appear: - assert world.browser.is_text_not_present(text, wait_time=5) + assert world.browser.is_text_not_present(text, wait_time=5*multiplier) else: - assert world.browser.is_text_present(text, wait_time=5) + assert world.browser.is_text_present(text, wait_time=5*multiplier) @step('I am logged in$') @@ -150,7 +153,7 @@ def i_am_logged_in(step): world.log_in(username='robot', password='test') world.browser.visit(django_url('/')) # You should not see the login link - assert_equals(world.browser.find_by_css('a#login'), []) + assert world.is_css_not_present('a#login') @step(u'I am an edX user$') diff --git a/jenkins/test_acceptance.sh b/jenkins/test_acceptance.sh index b7a244fe99..c2f2fa4ce3 100755 --- a/jenkins/test_acceptance.sh +++ b/jenkins/test_acceptance.sh @@ -30,10 +30,22 @@ TESTS_FAILED=0 # /usr/bin/Xvfb :1 -screen 0 1024x268x24 # This allows us to run Chrome without a display export DISPLAY=:1 +SKIP_TESTS="" + +# Testing for the existance of these environment variables +if [ ! -z ${LETTUCE_BROWSER+x} ]; then + SKIP_TESTS="--tag -skip_$LETTUCE_BROWSER" +fi +if [ ! -z ${SAUCE_ENABLED+x} ]; then + # SAUCE_INFO is a - seperated string PLATFORM-BROWSER-VERSION-DEVICE + # Error checking is done in the setting up of the browser + IFS='-' read -a SAUCE <<< "${SAUCE_INFO}" + SKIP_TESTS="--tag -skip_sauce --tag -skip_${SAUCE[1]}" +fi # Run the lms and cms acceptance tests # (the -v flag turns off color in the output) -rake test_acceptance_lms["-v 3"] || TESTS_FAILED=1 -rake test_acceptance_cms["-v 3"] || TESTS_FAILED=1 +rake test_acceptance_lms["-v 3 $SKIP_TESTS"] || TESTS_FAILED=1 +rake test_acceptance_cms["-v 3 $SKIP_TESTS"] || TESTS_FAILED=1 [ $TESTS_FAILED == '0' ] diff --git a/lms/djangoapps/courseware/features/login.feature b/lms/djangoapps/courseware/features/login.feature index 2b90c56f2d..4165a9bb9f 100644 --- a/lms/djangoapps/courseware/features/login.feature +++ b/lms/djangoapps/courseware/features/login.feature @@ -11,7 +11,8 @@ Feature: Login in as a registered user And I submit my credentials on the login form Then I should see the login error message "This account has not been activated" - # CHROME ONLY, firefox will not redirect properly + # firefox will not redirect properly when the whole suite is run + @skip_firefox Scenario: Login to an activated account Given I am an edX user And I am an activated user diff --git a/lms/djangoapps/courseware/features/signup.feature b/lms/djangoapps/courseware/features/signup.feature index 19dfd74f1c..3c9f491f7d 100644 --- a/lms/djangoapps/courseware/features/signup.feature +++ b/lms/djangoapps/courseware/features/signup.feature @@ -3,7 +3,8 @@ Feature: Sign in As a new user I want to signup for a student account - # CHROME ONLY, firefox will not redirect properly + # firefox will not redirect properly + @skip_firefox Scenario: Sign up from the homepage Given I visit the homepage When I click the link with the text "Register Now" diff --git a/lms/djangoapps/courseware/features/video.feature b/lms/djangoapps/courseware/features/video.feature index 74cd9cbcbb..6c8299f2c5 100644 --- a/lms/djangoapps/courseware/features/video.feature +++ b/lms/djangoapps/courseware/features/video.feature @@ -11,6 +11,8 @@ Feature: Video component Given the course has a Video component in Youtube mode Then when I view the video it has rendered in Youtube mode + # Firefox doesn't have HTML5 + @skip_firefox Scenario: Autoplay is enabled in LMS for a Video component Given the course has a Video component in HTML5 mode - Then when I view the video it has autoplay enabled \ No newline at end of file + Then when I view the video it has autoplay enabled diff --git a/lms/envs/acceptance.py b/lms/envs/acceptance.py index 1e188d3b45..cf64404161 100644 --- a/lms/envs/acceptance.py +++ b/lms/envs/acceptance.py @@ -8,6 +8,7 @@ so that we can run the lettuce acceptance tests. # pylint: disable=W0401, W0614 from .test import * +from .sauce import * # You need to start the server in debug mode, # otherwise the browser will not render the pages correctly @@ -17,7 +18,7 @@ DEBUG = True import logging logging.disable(logging.ERROR) import os -import random +from random import choice, randint def seed(): @@ -65,7 +66,7 @@ DATABASES = { # Set up XQueue information so that the lms will send # requests to a mock XQueue server running locally -XQUEUE_PORT = random.randint(1024, 65535) +XQUEUE_PORT = choice(PORTS) if SAUCE.get('SAUCE_ENABLED') else randint(1024, 65535) XQUEUE_INTERFACE = { "url": "http://127.0.0.1:%d" % XQUEUE_PORT, "django_auth": { @@ -93,5 +94,5 @@ FEEDBACK_SUBMISSION_EMAIL = 'dummy@example.com' # Include the lettuce app for acceptance testing, including the 'harvest' django-admin command INSTALLED_APPS += ('lettuce.django',) LETTUCE_APPS = ('courseware',) -LETTUCE_SERVER_PORT = random.randint(1024, 65535) -LETTUCE_BROWSER = 'chrome' +LETTUCE_SERVER_PORT = choice(PORTS) if SAUCE.get('SAUCE_ENABLED') else randint(1024, 65535) +LETTUCE_BROWSER = os.environ.get('LETTUCE_BROWSER', 'chrome') diff --git a/lms/envs/sauce.py b/lms/envs/sauce.py new file mode 100644 index 0000000000..8d48228251 --- /dev/null +++ b/lms/envs/sauce.py @@ -0,0 +1,63 @@ +""" +This config file extends the test environment configuration +so that we can run the lettuce acceptance tests on SauceLabs. +""" + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +import os + +PORTS = [2000, 2001, 2020, 2109, 2222, 2310, 3000, 3001, + 3030, 3210, 3333, 4000, 4001, 4040, 4321, 4502, 4503, + 5050, 5555, 5432, 6060, 6666, 6543, 7000, 7070, 7774, + 7777, 8003, 8031, 8080, 8081, 8765, 8888, + 9080, 9090, 9876, 9999, 49221, 55001] + +DESIRED_CAPABILITIES = { + 'chrome': DesiredCapabilities.CHROME, + 'internet explorer': DesiredCapabilities.INTERNETEXPLORER, + 'firefox': DesiredCapabilities.FIREFOX, + 'opera': DesiredCapabilities.OPERA, + 'iphone': DesiredCapabilities.IPHONE, + 'ipad': DesiredCapabilities.IPAD, + 'safari': DesiredCapabilities.SAFARI, + 'android': DesiredCapabilities.ANDROID +} + +# All keys must be URL and JSON encodable +# PLATFORM-BROWSER-VERSION_NUM-DEVICE +ALL_CONFIG = { + 'Linux-chrome--': ['Linux', 'chrome', '', ''], + 'Windows 8-chrome--': ['Windows 8', 'chrome', '', ''], + 'Windows 7-chrome--': ['Windows 7', 'chrome', '', ''], + 'Windows XP-chrome--': ['Windows XP', 'chrome', '', ''], + 'OS X 10.8-chrome--': ['OS X 10.8', 'chrome', '', ''], + 'OS X 10.6-chrome--': ['OS X 10.6', 'chrome', '', ''], + + 'Linux-firefox-23-': ['Linux', 'firefox', '23', ''], + 'Windows 8-firefox-23-': ['Windows 8', 'firefox', '23', ''], + 'Windows 7-firefox-23-': ['Windows 7', 'firefox', '23', ''], + 'Windows XP-firefox-23-': ['Windows XP', 'firefox', '23', ''], + + 'OS X 10.8-safari-6-': ['OS X 10.8', 'safari', '6', ''], + + 'Windows 8-internetexplorer-10-': ['Windows 8', 'internetexplorer', '10', ''], +} + +SAUCE_INFO = ALL_CONFIG.get(os.environ.get('SAUCE_INFO', 'Linux-chrome--')) + +# Information needed to utilize Sauce Labs. +SAUCE = { + 'SAUCE_ENABLED': os.environ.get('SAUCE_ENABLED'), + 'USERNAME': os.environ.get('SAUCE_USER_NAME'), + 'ACCESS_ID': os.environ.get('SAUCE_API_KEY'), + 'PLATFORM': SAUCE_INFO[0], + 'BROWSER': DESIRED_CAPABILITIES.get(SAUCE_INFO[1]), + 'VERSION': SAUCE_INFO[2], + 'DEVICE': SAUCE_INFO[3], + 'SESSION': 'Jenkins Acceptance Tests', + 'BUILD': os.environ.get('BUILD_DISPLAY_NAME', 'LETTUCE TESTS'), +}