From 1f0660469442cf7c79517ad35bf1b99f0f1fd0bd Mon Sep 17 00:00:00 2001 From: Jeremy Bowman Date: Wed, 16 Aug 2017 17:30:18 -0400 Subject: [PATCH] PLAT-1710 Support lettuce tests in Docker Devstack --- cms/envs/acceptance_docker.py | 61 ++++++++++++++ common/djangoapps/terrain/stubs/lti.py | 9 +- lms/djangoapps/courseware/features/events.py | 3 +- lms/djangoapps/courseware/features/lti.py | 3 +- lms/envs/acceptance_docker.py | 82 +++++++++++++++++++ pavelib/utils/test/suites/acceptance_suite.py | 17 ++-- requirements/edx/github.txt | 2 +- 7 files changed, 164 insertions(+), 13 deletions(-) create mode 100644 cms/envs/acceptance_docker.py create mode 100644 lms/envs/acceptance_docker.py diff --git a/cms/envs/acceptance_docker.py b/cms/envs/acceptance_docker.py new file mode 100644 index 0000000000..b00a94db5f --- /dev/null +++ b/cms/envs/acceptance_docker.py @@ -0,0 +1,61 @@ +""" +This config file extends the test environment configuration +so that we can run the lettuce acceptance tests. +""" + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=wildcard-import, unused-wildcard-import + +import os + +os.environ['EDXAPP_TEST_MONGO_HOST'] = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'edx.devstack.mongo') + +# noinspection PyUnresolvedReferences +from .acceptance import * + +update_module_store_settings( + MODULESTORE, + doc_store_settings={ + 'db': 'acceptance_xmodule', + 'host': MONGO_HOST, + 'port': MONGO_PORT_NUM, + 'collection': 'acceptance_modulestore_%s' % seed(), + }, + module_store_options={ + 'default_class': 'xmodule.raw_module.RawDescriptor', + 'fs_root': TEST_ROOT / "data", + }, + default_store=os.environ.get('DEFAULT_STORE', 'draft'), +) + +CONTENTSTORE = { + 'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore', + 'DOC_STORE_CONFIG': { + 'host': MONGO_HOST, + 'port': MONGO_PORT_NUM, + 'db': 'acceptance_xcontent_%s' % seed(), + }, + # allow for additional options that can be keyed on a name, e.g. 'trashcan' + 'ADDITIONAL_OPTIONS': { + 'trashcan': { + 'bucket': 'trash_fs' + } + } +} + +# Where to run: local, saucelabs, or grid +LETTUCE_SELENIUM_CLIENT = os.environ.get('LETTUCE_SELENIUM_CLIENT', 'grid') +SELENIUM_HOST = 'edx.devstack.{}'.format(LETTUCE_BROWSER) +SELENIUM_PORT = os.environ.get('SELENIUM_PORT', '4444') + +SELENIUM_GRID = { + 'URL': 'http://{}:{}/wd/hub'.format(SELENIUM_HOST, SELENIUM_PORT), + 'BROWSER': LETTUCE_BROWSER, +} + +# Point the URL used to test YouTube availability to our stub YouTube server +LETTUCE_HOST = os.environ['BOK_CHOY_HOSTNAME'] +YOUTUBE['API'] = "http://{}:{}/get_youtube_api/".format(LETTUCE_HOST, YOUTUBE_PORT) +YOUTUBE['METADATA_URL'] = "http://{}:{}/test_youtube/".format(LETTUCE_HOST, YOUTUBE_PORT) +YOUTUBE['TEXT_API']['url'] = "{}:{}/test_transcripts_youtube/".format(LETTUCE_HOST, YOUTUBE_PORT) diff --git a/common/djangoapps/terrain/stubs/lti.py b/common/djangoapps/terrain/stubs/lti.py index 7a545a837b..045cd7d9cd 100644 --- a/common/djangoapps/terrain/stubs/lti.py +++ b/common/djangoapps/terrain/stubs/lti.py @@ -18,6 +18,7 @@ from uuid import uuid4 import mock import oauthlib.oauth1 import requests +from django.conf import settings from http import StubHttpRequestHandler, StubHttpService from oauthlib.oauth1.rfc5849 import parameters, signature @@ -29,7 +30,7 @@ class StubLtiHandler(StubHttpRequestHandler): DEFAULT_CLIENT_KEY = 'test_client_key' DEFAULT_CLIENT_SECRET = 'test_client_secret' DEFAULT_LTI_ENDPOINT = 'correct_lti_endpoint' - DEFAULT_LTI_ADDRESS = 'http://127.0.0.1:{port}/' + DEFAULT_LTI_ADDRESS = 'http://{host}:{port}/' def do_GET(self): """ @@ -71,7 +72,8 @@ class StubLtiHandler(StubHttpRequestHandler): 'sourcedId': self.post_dict.get('lis_result_sourcedid') } - submit_url = '//{}:{}'.format(*self.server.server_address) + host = getattr(settings, 'LETTUCE_HOST', self.server.server_address[0]) + submit_url = '//{}:{}'.format(host, self.server.server_address[1]) content = self._create_content(status_message, submit_url) self.send_response(200, content) @@ -290,8 +292,9 @@ class StubLtiHandler(StubHttpRequestHandler): """ client_secret = unicode(self.server.config.get('client_secret', self.DEFAULT_CLIENT_SECRET)) + host = getattr(settings, 'LETTUCE_HOST', '127.0.0.1') port = self.server.server_address[1] - lti_base = self.DEFAULT_LTI_ADDRESS.format(port=port) + lti_base = self.DEFAULT_LTI_ADDRESS.format(host=host, port=port) lti_endpoint = self.server.config.get('lti_endpoint', self.DEFAULT_LTI_ENDPOINT) url = lti_base + lti_endpoint diff --git a/lms/djangoapps/courseware/features/events.py b/lms/djangoapps/courseware/features/events.py index c2528cec0e..f86b2c8544 100644 --- a/lms/djangoapps/courseware/features/events.py +++ b/lms/djangoapps/courseware/features/events.py @@ -1,5 +1,6 @@ # pylint: disable=missing-docstring +from django.conf import settings from lettuce import before, step, world from nose.tools import assert_equals, assert_in from pymongo import MongoClient @@ -19,7 +20,7 @@ REQUIRED_EVENT_FIELDS = [ @before.all def connect_to_mongodb(): - world.mongo_client = MongoClient() + world.mongo_client = MongoClient(host=settings.MONGO_HOST, port=settings.MONGO_PORT_NUM) world.event_collection = world.mongo_client['track']['events'] diff --git a/lms/djangoapps/courseware/features/lti.py b/lms/djangoapps/courseware/features/lti.py index b4f29972f3..b175e5802d 100644 --- a/lms/djangoapps/courseware/features/lti.py +++ b/lms/djangoapps/courseware/features/lti.py @@ -153,9 +153,10 @@ def set_incorrect_lti_passport(_step): @step(r'the course has an LTI component with (.*) fields(?:\:)?$') # , new_page is(.*), graded is(.*) def add_correct_lti_to_course(_step, fields): category = 'lti' + host = getattr(settings, 'LETTUCE_HOST', '127.0.0.1') metadata = { 'lti_id': 'correct_lti_id', - 'launch_url': 'http://127.0.0.1:{}/correct_lti_endpoint'.format(settings.LTI_PORT), + 'launch_url': 'http://{}:{}/correct_lti_endpoint'.format(host, settings.LTI_PORT), } if fields.strip() == 'incorrect_lti_id': # incorrect fields diff --git a/lms/envs/acceptance_docker.py b/lms/envs/acceptance_docker.py new file mode 100644 index 0000000000..6a9aa2da68 --- /dev/null +++ b/lms/envs/acceptance_docker.py @@ -0,0 +1,82 @@ +""" +This config file extends the test environment configuration +so that we can run the lettuce acceptance tests. +""" + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=wildcard-import, unused-wildcard-import + +import os + +os.environ['EDXAPP_TEST_MONGO_HOST'] = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'edx.devstack.mongo') + +# noinspection PyUnresolvedReferences +from .acceptance import * + +LETTUCE_HOST = os.environ['BOK_CHOY_HOSTNAME'] +SITE_NAME = '{}:{}'.format(LETTUCE_HOST, LETTUCE_SERVER_PORT) + +update_module_store_settings( + MODULESTORE, + doc_store_settings={ + 'db': 'acceptance_xmodule', + 'host': MONGO_HOST, + 'port': MONGO_PORT_NUM, + 'collection': 'acceptance_modulestore_%s' % seed(), + }, + module_store_options={ + 'fs_root': TEST_ROOT / "data", + }, + default_store=os.environ.get('DEFAULT_STORE', 'draft'), +) +CONTENTSTORE = { + 'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore', + 'DOC_STORE_CONFIG': { + 'host': MONGO_HOST, + 'port': MONGO_PORT_NUM, + 'db': 'acceptance_xcontent_%s' % seed(), + } +} + +TRACKING_BACKENDS.update({ + 'mongo': { + 'ENGINE': 'track.backends.mongodb.MongoBackend', + 'OPTIONS': { + 'database': 'test', + 'collection': 'events', + 'host': [ + 'edx.devstack.mongo' + ], + 'port': 27017 + } + } +}) + +EVENT_TRACKING_BACKENDS['tracking_logs']['OPTIONS']['backends'].update({ + 'mongo': { + 'ENGINE': 'eventtracking.backends.mongodb.MongoBackend', + 'OPTIONS': { + 'database': 'track', + 'host': [ + 'edx.devstack.mongo' + ], + 'port': 27017 + } + } +}) + +# Where to run: local, saucelabs, or grid +LETTUCE_SELENIUM_CLIENT = os.environ.get('LETTUCE_SELENIUM_CLIENT', 'grid') +SELENIUM_HOST = 'edx.devstack.{}'.format(LETTUCE_BROWSER) +SELENIUM_PORT = os.environ.get('SELENIUM_PORT', '4444') + +SELENIUM_GRID = { + 'URL': 'http://{}:{}/wd/hub'.format(SELENIUM_HOST, SELENIUM_PORT), + 'BROWSER': LETTUCE_BROWSER, +} + +# Point the URL used to test YouTube availability to our stub YouTube server +YOUTUBE['API'] = "http://{}:{}/get_youtube_api/".format(LETTUCE_HOST, YOUTUBE_PORT) +YOUTUBE['METADATA_URL'] = "http://{}:{}/test_youtube/".format(LETTUCE_HOST, YOUTUBE_PORT) +YOUTUBE['TEXT_API']['url'] = "{}:{}/test_transcripts_youtube/".format(LETTUCE_HOST, YOUTUBE_PORT) diff --git a/pavelib/utils/test/suites/acceptance_suite.py b/pavelib/utils/test/suites/acceptance_suite.py index e35ce3f2a1..e9e4c2e414 100644 --- a/pavelib/utils/test/suites/acceptance_suite.py +++ b/pavelib/utils/test/suites/acceptance_suite.py @@ -42,6 +42,7 @@ def setup_acceptance_db(): # Since we are using SQLLite, we can reset the database by deleting it on disk. DBS[db].remove() + settings = 'acceptance_docker' if Env.USING_DOCKER else 'acceptance' if all(DB_CACHES[cache].isfile() for cache in DB_CACHES.keys()): # To speed up migrations, we check for a cached database file and start from that. # The cached database file should be checked into the repo @@ -53,13 +54,13 @@ def setup_acceptance_db(): # Run migrations to update the db, starting from its cached state for db_alias in sorted(DBS.keys()): # pylint: disable=line-too-long - sh("./manage.py lms --settings acceptance migrate --traceback --noinput --fake-initial --database {}".format(db_alias)) - sh("./manage.py cms --settings acceptance migrate --traceback --noinput --fake-initial --database {}".format(db_alias)) + sh("./manage.py lms --settings {} migrate --traceback --noinput --fake-initial --database {}".format(settings, db_alias)) + sh("./manage.py cms --settings {} migrate --traceback --noinput --fake-initial --database {}".format(settings, db_alias)) else: # If no cached database exists, syncdb before migrating, then create the cache for db_alias in sorted(DBS.keys()): - sh("./manage.py lms --settings acceptance migrate --traceback --noinput --database {}".format(db_alias)) - sh("./manage.py cms --settings acceptance migrate --traceback --noinput --database {}".format(db_alias)) + sh("./manage.py lms --settings {} migrate --traceback --noinput --database {}".format(settings, db_alias)) + sh("./manage.py cms --settings {} migrate --traceback --noinput --database {}".format(settings, db_alias)) # Create the cache if it doesn't already exist for db_alias in DBS.keys(): @@ -77,6 +78,7 @@ class AcceptanceTest(TestSuite): self.system = kwargs.get('system') self.default_store = kwargs.get('default_store') self.extra_args = kwargs.get('extra_args', '') + self.settings = 'acceptance_docker' if Env.USING_DOCKER else 'acceptance' def __enter__(self): super(AcceptanceTest, self).__enter__() @@ -91,15 +93,16 @@ class AcceptanceTest(TestSuite): @property def cmd(self): + lettuce_host = ['LETTUCE_HOST={}'.format(Env.SERVER_HOST)] if Env.USING_DOCKER else [] report_file = self.report_dir / "{}.xml".format(self.system) report_args = ["--xunit-file {}".format(report_file)] - return [ + return lettuce_host + [ # set DBUS_SESSION_BUS_ADDRESS to avoid hangs on Chrome "DBUS_SESSION_BUS_ADDRESS=/dev/null", "DEFAULT_STORE={}".format(self.default_store), "./manage.py", self.system, - "--settings=acceptance", + "--settings={}".format(self.settings), "harvest", "--traceback", "--debug-mode", @@ -112,7 +115,7 @@ class AcceptanceTest(TestSuite): """ Internal helper method to manage asset compilation """ - args = [self.system, '--settings=acceptance'] + args = [self.system, '--settings={}'.format(self.settings)] call_task('pavelib.assets.update_assets', args=args) diff --git a/requirements/edx/github.txt b/requirements/edx/github.txt index 815dc3a123..fba7c387ce 100644 --- a/requirements/edx/github.txt +++ b/requirements/edx/github.txt @@ -68,7 +68,7 @@ git+https://github.com/hmarr/django-debug-toolbar-mongo.git@b0686a76f1ce3532088c git+https://github.com/edx/rfc6266.git@v0.0.5-edx#egg=rfc6266==0.0.5-edx # Used for testing -git+https://github.com/edx/lettuce.git@0.2.20.002#egg=lettuce==0.2.20.002 +git+https://github.com/edx/lettuce.git@31b0dfd865766243e9b563ec65fae9122edf7975#egg=lettuce==0.2.23+edx.1 # Why a DRF fork? See: https://openedx.atlassian.net/browse/PLAT-1581 git+https://github.com/edx/django-rest-framework.git@1ceda7c086fddffd1c440cc86856441bbf0bd9cb#egg=djangorestframework==3.6.3