diff --git a/cms/envs/bok_choy.py b/cms/envs/bok_choy.py index 7919c764a3..9c69e46020 100644 --- a/cms/envs/bok_choy.py +++ b/cms/envs/bok_choy.py @@ -21,7 +21,7 @@ from openedx.core.release import RELEASE_LINE # Unlike in prod, we use the JSON files stored in this repo. # This is a convenience for ensuring (a) that we can consistently find the files # and (b) that the files are the same in Jenkins as in local dev. -os.environ['SERVICE_VARIANT'] = 'bok_choy' +os.environ['SERVICE_VARIANT'] = 'bok_choy_docker' if 'BOK_CHOY_HOSTNAME' in os.environ else 'bok_choy' os.environ['CONFIG_ROOT'] = path(__file__).abspath().dirname() from .aws import * # pylint: disable=wildcard-import, unused-wildcard-import diff --git a/cms/envs/bok_choy_docker.auth.json b/cms/envs/bok_choy_docker.auth.json new file mode 100644 index 0000000000..6cc9885c45 --- /dev/null +++ b/cms/envs/bok_choy_docker.auth.json @@ -0,0 +1,113 @@ +{ + "ANALYTICS_API_KEY": "", + "AWS_ACCESS_KEY_ID": "", + "AWS_SECRET_ACCESS_KEY": "", + "CELERY_BROKER_PASSWORD": "celery", + "CELERY_BROKER_USER": "celery", + "CONTENTSTORE": { + "DOC_STORE_CONFIG": { + "collection": "modulestore", + "db": "test", + "host": [ + "edx.devstack.mongo" + ], + "port": 27017 + }, + "ENGINE": "xmodule.contentstore.mongo.MongoContentStore", + "OPTIONS": { + "db": "test", + "host": [ + "edx.devstack.mongo" + ], + "port": 27017 + } + }, + "DATABASES": { + "default": { + "ENGINE": "django.db.backends.mysql", + "HOST": "edx.devstack.mysql", + "NAME": "edxtest", + "PASSWORD": "", + "PORT": "3306", + "USER": "root" + }, + "student_module_history": { + "ENGINE": "django.db.backends.mysql", + "HOST": "edx.devstack.mysql", + "NAME": "student_module_history_test", + "PASSWORD": "", + "PORT": "3306", + "USER": "root" + } + }, + "DOC_STORE_CONFIG": { + "collection": "modulestore", + "db": "test", + "host": [ + "edx.devstack.mongo" + ], + "port": 27017 + }, + "MODULESTORE": { + "default": { + "ENGINE": "xmodule.modulestore.mixed.MixedModuleStore", + "OPTIONS": { + "mappings": {}, + "stores": [ + { + "NAME": "draft", + "DOC_STORE_CONFIG": { + "collection": "modulestore", + "db": "test", + "host": [ + "edx.devstack.mongo" + ], + "port": 27017 + }, + "ENGINE": "xmodule.modulestore.mongo.DraftMongoModuleStore", + "OPTIONS": { + "collection": "modulestore", + "db": "test", + "default_class": "xmodule.hidden_module.HiddenDescriptor", + "fs_root": "** OVERRIDDEN **", + "host": [ + "edx.devstack.mongo" + ], + "port": 27017, + "render_template": "edxmako.shortcuts.render_to_string" + } + }, + { + "NAME": "xml", + "ENGINE": "xmodule.modulestore.xml.XMLModuleStore", + "OPTIONS": { + "data_dir": "** OVERRIDDEN **", + "default_class": "xmodule.hidden_module.HiddenDescriptor" + } + } + ] + } + } + }, + "DJFS": { + "type": "s3fs", + "bucket": "test", + "prefix": "test", + "aws_access_key_id": "test", + "aws_secret_access_key": "test" + }, + "SECRET_KEY": "", + "XQUEUE_INTERFACE": { + "basic_auth": [ + "edx", + "edx" + ], + "django_auth": { + "password": "password", + "username": "lms" + }, + "url": "http://localhost:18040" + }, + "ZENDESK_API_KEY": "", + "ZENDESK_USER": "" +} diff --git a/cms/envs/bok_choy_docker.env.json b/cms/envs/bok_choy_docker.env.json new file mode 100644 index 0000000000..6e49423f36 --- /dev/null +++ b/cms/envs/bok_choy_docker.env.json @@ -0,0 +1,109 @@ +{ + "ANALYTICS_SERVER_URL": "", + "BOOK_URL": "", + "BUGS_EMAIL": "bugs@example.com", + "BULK_EMAIL_DEFAULT_FROM_EMAIL": "no-reply@example.com", + "CACHES": { + "celery": { + "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", + "KEY_FUNCTION": "util.memcache.safe_key", + "KEY_PREFIX": "integration_celery", + "LOCATION": [ + "edx.devstack.memcached:11211" + ] + }, + "default": { + "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", + "KEY_FUNCTION": "util.memcache.safe_key", + "KEY_PREFIX": "sandbox_default", + "LOCATION": [ + "edx.devstack.memcached:11211" + ] + }, + "general": { + "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", + "KEY_FUNCTION": "util.memcache.safe_key", + "KEY_PREFIX": "sandbox_general", + "LOCATION": [ + "edx.devstack.memcached:11211" + ] + }, + "mongo_metadata_inheritance": { + "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", + "KEY_FUNCTION": "util.memcache.safe_key", + "KEY_PREFIX": "integration_mongo_metadata_inheritance", + "LOCATION": [ + "edx.devstack.memcached:11211" + ] + }, + "staticfiles": { + "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", + "KEY_FUNCTION": "util.memcache.safe_key", + "KEY_PREFIX": "integration_static_files", + "LOCATION": [ + "edx.devstack.memcached:11211" + ] + } + }, + "CELERY_ALWAYS_EAGER": true, + "CELERY_BROKER_HOSTNAME": "localhost", + "CELERY_BROKER_TRANSPORT": "amqp", + "CERT_QUEUE": "certificates", + "CMS_BASE": "** OVERRIDDEN **", + "CODE_JAIL": { + "limits": { + "REALTIME": 3, + "VMEM": 0 + } + }, + "COMMENTS_SERVICE_KEY": "password", + "COMMENTS_SERVICE_URL": "http://localhost:4567", + "CONTACT_EMAIL": "info@example.com", + "DEFAULT_FEEDBACK_EMAIL": "feedback@example.com", + "DEFAULT_FROM_EMAIL": "registration@example.com", + "EMAIL_BACKEND": "django.core.mail.backends.smtp.EmailBackend", + "SOCIAL_SHARING_SETTINGS": { + "CUSTOM_COURSE_URLS": true + }, + "FEATURES": { + "AUTH_USE_OPENID_PROVIDER": true, + "CERTIFICATES_HTML_VIEW": true, + "ENABLE_DISCUSSION_SERVICE": true, + "ENABLE_GRADE_DOWNLOADS": true, + "ENTRANCE_EXAMS": true, + "MILESTONES_APP": true, + "PREVIEW_LMS_BASE": "preview.localhost:8003", + "ENABLE_CONTENT_LIBRARIES": true, + "ENABLE_SPECIAL_EXAMS": true, + "SHOW_HEADER_LANGUAGE_SELECTOR": true, + "ENABLE_EXTENDED_COURSE_DETAILS": true, + "CUSTOM_COURSES_EDX": true + }, + "FEEDBACK_SUBMISSION_EMAIL": "", + "GITHUB_REPO_ROOT": "** OVERRIDDEN **", + "JWT_AUTH": { + "JWT_SECRET_KEY": "super-secret-key" + }, + "GRADES_DOWNLOAD": { + "BUCKET": "edx-grades", + "ROOT_PATH": "/tmp/edx-s3/grades", + "STORAGE_TYPE": "localfs" + }, + "LMS_BASE": "** OVERRIDDEN **", + "LMS_ROOT_URL": "** OVERRIDDEN **", + "LOCAL_LOGLEVEL": "INFO", + "LOGGING_ENV": "sandbox", + "LOG_DIR": "** OVERRIDDEN **", + "MEDIA_URL": "/media/", + "MKTG_URL_LINK_MAP": {}, + "PLATFORM_NAME": "édX", + "SERVER_EMAIL": "devops@example.com", + "SESSION_COOKIE_DOMAIN": null, + "SITE_NAME": "localhost", + "STATIC_URL_BASE": "/static/", + "SYSLOG_SERVER": "", + "TECH_SUPPORT_EMAIL": "technical@example.com", + "TIME_ZONE": "America/New_York", + "WIKI_ENABLED": true, + "OAUTH_OIDC_ISSUER": "https://www.example.com/oauth2" +} diff --git a/cms/envs/bok_choy_docker.py b/cms/envs/bok_choy_docker.py new file mode 100644 index 0000000000..96dd3902cc --- /dev/null +++ b/cms/envs/bok_choy_docker.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +""" +Settings for Bok Choy tests that are used when running Studio in Docker-based devstack. +""" + +# noinspection PyUnresolvedReferences +from .bok_choy import * # pylint: disable=wildcard-import + +CMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ['BOK_CHOY_CMS_PORT']) +LMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ['BOK_CHOY_LMS_PORT']) +LMS_ROOT_URL = 'http://{}'.format(LMS_BASE) + +# Docker does not support the syslog socket at /dev/log. Rely on the console. +LOGGING['handlers']['local'] = LOGGING['handlers']['tracking'] = { + 'class': 'logging.NullHandler', +} + +LOGGING['loggers']['tracking']['handlers'] = ['console'] diff --git a/common/test/acceptance/fixtures/__init__.py b/common/test/acceptance/fixtures/__init__.py index ae494659e3..4e4065ba45 100644 --- a/common/test/acceptance/fixtures/__init__.py +++ b/common/test/acceptance/fixtures/__init__.py @@ -1,11 +1,15 @@ import os +HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', 'localhost') +CMS_PORT = os.environ.get('BOK_CHOY_CMS_PORT', '8031') +LMS_PORT = os.environ.get('BOK_CHOY_LMS_PORT', '8003') + # Get the URL of the Studio instance under test -STUDIO_BASE_URL = os.environ.get('studio_url', 'http://localhost:8031') +STUDIO_BASE_URL = os.environ.get('studio_url', 'http://{}:{}'.format(HOSTNAME, CMS_PORT)) # Get the URL of the LMS instance under test -LMS_BASE_URL = os.environ.get('lms_url', 'http://localhost:8003') +LMS_BASE_URL = os.environ.get('lms_url', 'http://{}:{}'.format(HOSTNAME, LMS_PORT)) # Get the URL of the XQueue stub used in the test XQUEUE_STUB_URL = os.environ.get('xqueue_url', 'http://localhost:8040') diff --git a/common/test/acceptance/pages/common/__init__.py b/common/test/acceptance/pages/common/__init__.py index 488c0dbe63..264cba811d 100644 --- a/common/test/acceptance/pages/common/__init__.py +++ b/common/test/acceptance/pages/common/__init__.py @@ -1,7 +1,11 @@ import os +HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', 'localhost') +CMS_PORT = os.environ.get('BOK_CHOY_CMS_PORT', 8031) +LMS_PORT = os.environ.get('BOK_CHOY_CMS_PORT', 8003) + # Get the URL of the instance under test -BASE_URL = os.environ.get('test_url', 'http://localhost:8003') +BASE_URL = os.environ.get('test_url', 'http://{}:{}'.format(HOSTNAME, LMS_PORT)) # The URL used for user auth in testing -AUTH_BASE_URL = os.environ.get('test_url', 'http://localhost:8031') +AUTH_BASE_URL = os.environ.get('test_url', 'http://{}:{}'.format(HOSTNAME, CMS_PORT)) diff --git a/common/test/acceptance/pages/common/auto_auth.py b/common/test/acceptance/pages/common/auto_auth.py index 851baf9cbb..8e265dd842 100644 --- a/common/test/acceptance/pages/common/auto_auth.py +++ b/common/test/acceptance/pages/common/auto_auth.py @@ -8,7 +8,9 @@ import urllib from bok_choy.page_object import XSS_INJECTION, PageObject, unguarded # The URL used for user auth in testing -AUTH_BASE_URL = os.environ.get('test_url', 'http://localhost:8031') +HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', 'localhost') +CMS_PORT = os.environ.get('BOK_CHOY_CMS_PORT', 8031) +AUTH_BASE_URL = os.environ.get('test_url', 'http://{}:{}'.format(HOSTNAME, CMS_PORT)) class AutoAuthPage(PageObject): diff --git a/common/test/acceptance/pages/lms/__init__.py b/common/test/acceptance/pages/lms/__init__.py index 1a22747f06..76a1b94530 100644 --- a/common/test/acceptance/pages/lms/__init__.py +++ b/common/test/acceptance/pages/lms/__init__.py @@ -1,4 +1,6 @@ import os # Get the URL of the instance under test -BASE_URL = os.environ.get('test_url', 'http://localhost:8003') +HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', 'localhost') +LMS_PORT = os.environ.get('BOK_CHOY_LMS_PORT', 8003) +BASE_URL = os.environ.get('test_url', 'http://{}:{}'.format(HOSTNAME, LMS_PORT)) diff --git a/common/test/acceptance/pages/studio/__init__.py b/common/test/acceptance/pages/studio/__init__.py index 1965a7f353..9cda972626 100644 --- a/common/test/acceptance/pages/studio/__init__.py +++ b/common/test/acceptance/pages/studio/__init__.py @@ -1,4 +1,6 @@ import os # Get the URL of the instance under test -BASE_URL = os.environ.get('test_url', 'http://localhost:8031') +HOSTNAME = os.environ.get('BOK_CHOY_HOSTNAME', 'localhost') +CMS_PORT = os.environ.get('BOK_CHOY_CMS_PORT', 8031) +BASE_URL = os.environ.get('test_url', 'http://{}:{}'.format(HOSTNAME, CMS_PORT)) diff --git a/lms/envs/bok_choy.py b/lms/envs/bok_choy.py index 2d92f01c1b..461907c34e 100644 --- a/lms/envs/bok_choy.py +++ b/lms/envs/bok_choy.py @@ -25,7 +25,7 @@ TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root" # Unlike in prod, we use the JSON files stored in this repo. # This is a convenience for ensuring (a) that we can consistently find the files # and (b) that the files are the same in Jenkins as in local dev. -os.environ['SERVICE_VARIANT'] = 'bok_choy' +os.environ['SERVICE_VARIANT'] = 'bok_choy_docker' if 'BOK_CHOY_HOSTNAME' in os.environ else 'bok_choy' os.environ['CONFIG_ROOT'] = CONFIG_ROOT from .aws import * # pylint: disable=wildcard-import, unused-wildcard-import diff --git a/lms/envs/bok_choy_docker.auth.json b/lms/envs/bok_choy_docker.auth.json new file mode 100644 index 0000000000..f37b564a27 --- /dev/null +++ b/lms/envs/bok_choy_docker.auth.json @@ -0,0 +1,148 @@ +{ + "ANALYTICS_API_KEY": "", + "AWS_ACCESS_KEY_ID": "", + "AWS_SECRET_ACCESS_KEY": "", + "CC_PROCESSOR_NAME": "CyberSource2", + "CC_PROCESSOR": { + "CyberSource2": { + "SECRET_KEY": "abcd123", + "ACCESS_KEY": "abcd123", + "PROFILE_ID": "edx", + "PURCHASE_ENDPOINT": "/shoppingcart/payment_fake" + } + }, + "CELERY_BROKER_PASSWORD": "celery", + "CELERY_BROKER_USER": "celery", + "CONTENTSTORE": { + "DOC_STORE_CONFIG": { + "collection": "modulestore", + "db": "test", + "host": [ + "edx.devstack.mongo" + ], + "port": 27017 + }, + "ENGINE": "xmodule.contentstore.mongo.MongoContentStore", + "OPTIONS": { + "db": "test", + "host": [ + "edx.devstack.mongo" + ], + "port": 27017 + } + }, + "DATABASES": { + "default": { + "ENGINE": "django.db.backends.mysql", + "HOST": "edx.devstack.mysql", + "NAME": "edxtest", + "PASSWORD": "", + "PORT": "3306", + "USER": "root" + }, + "student_module_history": { + "ENGINE": "django.db.backends.mysql", + "HOST": "edx.devstack.mysql", + "NAME": "student_module_history_test", + "PASSWORD": "", + "PORT": "3306", + "USER": "root" + } + }, + "DOC_STORE_CONFIG": { + "collection": "modulestore", + "db": "test", + "host": [ + "edx.devstack.mongo" + ], + "port": 27017 + }, + "TRACKING_BACKENDS": { + "mongo": { + "ENGINE": "track.backends.mongodb.MongoBackend", + "OPTIONS": { + "database": "test", + "collection": "events", + "host": [ + "edx.devstack.mongo" + ], + "port": 27017 + } + } + }, + "EVENT_TRACKING_BACKENDS": { + "mongo": { + "ENGINE": "eventtracking.backends.mongodb.MongoBackend", + "OPTIONS": { + "database": "test", + "collection": "events", + "host": [ + "edx.devstack.mongo" + ], + "port": 27017 + } + } + }, + "MODULESTORE": { + "default": { + "ENGINE": "xmodule.modulestore.mixed.MixedModuleStore", + "OPTIONS": { + "mappings": {}, + "stores": [ + { + "NAME": "draft", + "DOC_STORE_CONFIG": { + "collection": "modulestore", + "db": "test", + "host": [ + "edx.devstack.mongo" + ], + "port": 27017 + }, + "ENGINE": "xmodule.modulestore.mongo.DraftMongoModuleStore", + "OPTIONS": { + "collection": "modulestore", + "db": "test", + "default_class": "xmodule.hidden_module.HiddenDescriptor", + "fs_root": "** OVERRIDDEN **", + "host": [ + "edx.devstack.mongo" + ], + "port": 27017, + "render_template": "edxmako.shortcuts.render_to_string" + } + }, + { + "NAME": "xml", + "ENGINE": "xmodule.modulestore.xml.XMLModuleStore", + "OPTIONS": { + "data_dir": "** OVERRIDDEN **", + "default_class": "xmodule.hidden_module.HiddenDescriptor" + } + } + ] + } + } + }, + "SECRET_KEY": "", + "DJFS": { + "type": "s3fs", + "bucket": "test", + "prefix": "test", + "aws_access_key_id": "test", + "aws_secret_access_key": "test" + }, + "XQUEUE_INTERFACE": { + "basic_auth": [ + "edx", + "edx" + ], + "django_auth": { + "password": "password", + "username": "lms" + }, + "url": "** OVERRIDDEN **" + }, + "ZENDESK_API_KEY": "", + "ZENDESK_USER": "" +} diff --git a/lms/envs/bok_choy_docker.env.json b/lms/envs/bok_choy_docker.env.json new file mode 100644 index 0000000000..b3323285d9 --- /dev/null +++ b/lms/envs/bok_choy_docker.env.json @@ -0,0 +1,153 @@ +{ + "ANALYTICS_SERVER_URL": "", + "ANALYTICS_DASHBOARD_URL": "", + "BOOK_URL": "", + "BUGS_EMAIL": "bugs@example.com", + "BULK_EMAIL_DEFAULT_FROM_EMAIL": "no-reply@example.com", + "CACHES": { + "celery": { + "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", + "KEY_FUNCTION": "util.memcache.safe_key", + "KEY_PREFIX": "integration_celery", + "LOCATION": [ + "edx.devstack.memcached:11211" + ] + }, + "default": { + "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", + "KEY_FUNCTION": "util.memcache.safe_key", + "KEY_PREFIX": "sandbox_default", + "LOCATION": [ + "edx.devstack.memcached:11211" + ] + }, + "general": { + "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", + "KEY_FUNCTION": "util.memcache.safe_key", + "KEY_PREFIX": "sandbox_general", + "LOCATION": [ + "edx.devstack.memcached:11211" + ] + }, + "mongo_metadata_inheritance": { + "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", + "KEY_FUNCTION": "util.memcache.safe_key", + "KEY_PREFIX": "integration_mongo_metadata_inheritance", + "LOCATION": [ + "edx.devstack.memcached:11211" + ] + }, + "staticfiles": { + "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", + "KEY_FUNCTION": "util.memcache.safe_key", + "KEY_PREFIX": "integration_static_files", + "LOCATION": [ + "edx.devstack.memcached:11211" + ] + } + }, + "CELERY_BROKER_HOSTNAME": "localhost", + "CELERY_BROKER_TRANSPORT": "amqp", + "CERT_QUEUE": "certificates", + "CMS_BASE": "** OVERRIDDEN **", + "CODE_JAIL": { + "limits": { + "REALTIME": 3, + "VMEM": 0 + } + }, + "COMMENTS_SERVICE_KEY": "password", + "COMMENTS_SERVICE_URL": "http://localhost:4567", + "CONTACT_EMAIL": "info@example.com", + "DEFAULT_FEEDBACK_EMAIL": "feedback@example.com", + "DEFAULT_FROM_EMAIL": "registration@example.com", + "EMAIL_BACKEND": "django.core.mail.backends.dummy.EmailBackend", + "SOCIAL_SHARING_SETTINGS": { + "CUSTOM_COURSE_URLS": true, + "DASHBOARD_FACEBOOK": true, + "CERTIFICATE_FACEBOOK": true, + "CERTIFICATE_FACEBOOK_TEXT": "Testing facebook feature:", + "DASHBOARD_TWITTER": true, + "DASHBOARD_TWITTER_TEXT": "Testing feature:" + }, + "FEATURES": { + "AUTH_USE_OPENID_PROVIDER": true, + "CERTIFICATES_HTML_VIEW": true, + "CERTIFICATES_INSTRUCTOR_GENERATION": true, + "ENABLE_PAYMENT_FAKE": true, + "ENABLE_VERIFIED_CERTIFICATES": true, + "ENABLE_DISCUSSION_SERVICE": true, + "ENABLE_GRADE_DOWNLOADS": true, + "ENABLE_THIRD_PARTY_AUTH": true, + "ENABLE_COMBINED_LOGIN_REGISTRATION": true, + "PREVIEW_LMS_BASE": "preview.localhost:8003", + "ALLOW_AUTOMATED_SIGNUPS": true, + "AUTOMATIC_AUTH_FOR_TESTING": true, + "MODE_CREATION_FOR_TESTING": true, + "EXPOSE_CACHE_PROGRAMS_ENDPOINT": true, + "AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING": true, + "ENABLE_COURSE_DISCOVERY": true, + "ENABLE_SPECIAL_EXAMS": true, + "SHOW_HEADER_LANGUAGE_SELECTOR": true, + "CUSTOM_COURSES_EDX": true + }, + "FEEDBACK_SUBMISSION_EMAIL": "", + "GITHUB_REPO_ROOT": "** OVERRIDDEN **", + "JWT_AUTH": { + "JWT_SECRET_KEY": "super-secret-key" + }, + "LMS_BASE": "** OVERRIDDEN **", + "LMS_ROOT_URL": "** OVERRIDDEN **", + "LOCAL_LOGLEVEL": "INFO", + "LOGGING_ENV": "sandbox", + "LOG_DIR": "** OVERRIDDEN **", + "MEDIA_URL": "/media/", + "MKTG_URL_LINK_MAP": { + "ABOUT": "about", + "PRIVACY": "privacy", + "TOS": "tos", + "WHAT_IS_VERIFIED_CERT": "verified-certificate", + "COURSES": "courses", + "CONTACT": "contact", + "NEWS": "news", + "HONOR": "honor", + "CAREERS": "careers", + "HELP_CENTER": "help-center", + "BLOG": "blog", + "PRESS": "press", + "DONATE": "donate", + "ROOT": "root", + "SITEMAP.XML": "sitemap_xml" + }, + "PLATFORM_NAME": "édX", + "REGISTRATION_EXTENSION_FORM": "openedx.core.djangoapps.user_api.tests.test_helpers.TestCaseForm", + "REGISTRATION_EXTRA_FIELDS": { + "level_of_education": "optional", + "gender": "optional", + "year_of_birth": "optional", + "mailing_address": "optional", + "goals": "optional", + "honor_code": "required", + "terms_of_service": "hidden", + "city": "hidden", + "country": "required" + }, + "SERVER_EMAIL": "devops@example.com", + "SESSION_COOKIE_DOMAIN": null, + "SITE_NAME": "localhost:8003", + "STATIC_URL_BASE": "/static/", + "SUPPORT_SITE_LINK": "https://support.example.com", + "PASSWORD_RESET_SUPPORT_LINK": "https://support.example.com/password-reset-help.html", + "ACTIVATION_EMAIL_SUPPORT_LINK": "https://support.example.com/activation-email-help.html", + "SYSLOG_SERVER": "", + "TECH_SUPPORT_EMAIL": "technical@example.com", + "THIRD_PARTY_AUTH_BACKENDS": [ + "social.backends.google.GoogleOAuth2", + "social.backends.linkedin.LinkedinOAuth2", + "social.backends.facebook.FacebookOAuth2", + "third_party_auth.dummy.DummyBackend", + "third_party_auth.saml.SAMLAuthBackend" + ], + "TIME_ZONE": "America/New_York", + "WIKI_ENABLED": true +} diff --git a/lms/envs/bok_choy_docker.py b/lms/envs/bok_choy_docker.py new file mode 100644 index 0000000000..96dd3902cc --- /dev/null +++ b/lms/envs/bok_choy_docker.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +""" +Settings for Bok Choy tests that are used when running Studio in Docker-based devstack. +""" + +# noinspection PyUnresolvedReferences +from .bok_choy import * # pylint: disable=wildcard-import + +CMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ['BOK_CHOY_CMS_PORT']) +LMS_BASE = '{}:{}'.format(os.environ['BOK_CHOY_HOSTNAME'], os.environ['BOK_CHOY_LMS_PORT']) +LMS_ROOT_URL = 'http://{}'.format(LMS_BASE) + +# Docker does not support the syslog socket at /dev/log. Rely on the console. +LOGGING['handlers']['local'] = LOGGING['handlers']['tracking'] = { + 'class': 'logging.NullHandler', +} + +LOGGING['loggers']['tracking']['handlers'] = ['console'] diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index 700032e30b..f762aad887 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -65,13 +65,20 @@ class Env(object): # Directory that videos are served from VIDEO_SOURCE_DIR = REPO_ROOT / "test_root" / "data" / "video" + # Detect if in a Docker container, and if so which one + SERVER_HOST = os.environ.get('BOK_CHOY_HOSTNAME', '0.0.0.0') + USING_DOCKER = SERVER_HOST != '0.0.0.0' + SETTINGS = 'bok_choy_docker' if USING_DOCKER else 'bok_choy' + BOK_CHOY_SERVERS = { 'lms': { - 'port': 8003, + 'host': SERVER_HOST, + 'port': os.environ.get('BOK_CHOY_LMS_PORT', '8003'), 'log': BOK_CHOY_LOG_DIR / "bok_choy_lms.log" }, 'cms': { - 'port': 8031, + 'host': SERVER_HOST, + 'port': os.environ.get('BOK_CHOY_CMS_PORT', '8031'), 'log': BOK_CHOY_LOG_DIR / "bok_choy_studio.log" } } @@ -123,8 +130,10 @@ class Env(object): } # Mongo databases that will be dropped before/after the tests run + BOK_CHOY_MONGO_HOST = 'edx.devstack.mongo' if USING_DOCKER else 'localhost' BOK_CHOY_MONGO_DATABASE = "test" - BOK_CHOY_CACHE = memcache.Client(['0.0.0.0:11211'], debug=0) + BOK_CHOY_CACHE_HOST = 'edx.devstack.memcached' if USING_DOCKER else '0.0.0.0' + BOK_CHOY_CACHE = memcache.Client(['{}:11211'.format(BOK_CHOY_CACHE_HOST)], debug=0) # Test Ids Directory TEST_DIR = REPO_ROOT / ".testids" diff --git a/pavelib/utils/test/bokchoy_utils.py b/pavelib/utils/test/bokchoy_utils.py index d3df89c21e..0aaf72eafb 100644 --- a/pavelib/utils/test/bokchoy_utils.py +++ b/pavelib/utils/test/bokchoy_utils.py @@ -44,11 +44,12 @@ def start_servers(options): cmd = ( "DEFAULT_STORE={default_store} " "coverage run --rcfile={coveragerc} -m " - "manage {service} --settings bok_choy runserver " + "manage {service} --settings {settings} runserver " "{address} --traceback --noreload".format( default_store=options.default_store, coveragerc=coveragerc, service=service, + settings=Env.SETTINGS, address=address, ) ) @@ -83,7 +84,7 @@ def wait_for_server(server, port): attempts = 0 server_ok = False - while attempts < 30: + while attempts < 120: try: connection = httplib.HTTPConnection(server, port, timeout=10) connection.request('GET', '/') @@ -107,7 +108,7 @@ def wait_for_test_servers(): """ for service, info in Env.BOK_CHOY_SERVERS.iteritems(): - ready = wait_for_server("0.0.0.0", info['port']) + ready = wait_for_server(info['host'], info['port']) if not ready: msg = colorize( "red", @@ -123,7 +124,7 @@ def is_mongo_running(): """ # The mongo command will connect to the service, # failing with a non-zero exit code if it cannot connect. - output = os.popen('mongo --eval "print(\'running\')"').read() + output = os.popen('mongo --host {} --eval "print(\'running\')"'.format(Env.BOK_CHOY_MONGO_HOST)).read() return output and "running" in output @@ -155,7 +156,8 @@ def clear_mongo(): Clears mongo database. """ sh( - "mongo {} --eval 'db.dropDatabase()' > /dev/null".format( + "mongo --host {} {} --eval 'db.dropDatabase()' > /dev/null".format( + Env.BOK_CHOY_MONGO_HOST, Env.BOK_CHOY_MONGO_DATABASE, ) ) @@ -191,6 +193,9 @@ def check_mysql(): """ Check that mysql is running """ + if 'BOK_CHOY_HOSTNAME' in os.environ: + # mysql should be running in a separate Docker container + return if not is_mysql_running(): msg = colorize('red', "MySQL is not running locally.") print msg diff --git a/pavelib/utils/test/suites/bokchoy_suite.py b/pavelib/utils/test/suites/bokchoy_suite.py index 6d0cde97f9..6c05afe93c 100644 --- a/pavelib/utils/test/suites/bokchoy_suite.py +++ b/pavelib/utils/test/suites/bokchoy_suite.py @@ -48,9 +48,10 @@ def load_bok_choy_data(options): print 'Loading data from json fixtures in db_fixtures directory' sh( "DEFAULT_STORE={default_store}" - " ./manage.py lms --settings bok_choy loaddata --traceback" + " ./manage.py lms --settings {settings} loaddata --traceback" " common/test/db_fixtures/*.json".format( default_store=options.default_store, + settings=Env.SETTINGS ) ) @@ -76,9 +77,10 @@ def load_courses(options): sh( "DEFAULT_STORE={default_store}" - " ./manage.py cms --settings=bok_choy import {import_dir}".format( + " ./manage.py cms --settings={settings} import {import_dir}".format( default_store=options.default_store, - import_dir=options.imports_dir + import_dir=options.imports_dir, + settings=Env.SETTINGS ) ) else: @@ -261,7 +263,7 @@ class BokChoyTestSuite(TestSuite): # Clean up data we created in the databases msg = colorize('green', "Cleaning up databases...") print msg - sh("./manage.py lms --settings bok_choy flush --traceback --noinput") + sh("./manage.py lms --settings {settings} flush --traceback --noinput".format(settings=Env.SETTINGS)) clear_mongo() @property diff --git a/pavelib/utils/test/utils.py b/pavelib/utils/test/utils.py index d6aed364a6..63ece4d0ab 100644 --- a/pavelib/utils/test/utils.py +++ b/pavelib/utils/test/utils.py @@ -10,6 +10,8 @@ from paver.easy import cmdopts, sh, task from pavelib.utils.envs import Env from pavelib.utils.timer import timed +from bok_choy.browser import browser + MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017')) MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost') MINIMUM_FIREFOX_VERSION = 28.0 @@ -78,6 +80,33 @@ def check_firefox_version(): """ Check that firefox is the correct version. """ + if 'BOK_CHOY_HOSTNAME' in os.environ: + # Firefox is running in a separate Docker container; get its version via Selenium + driver = browser() + capabilities = driver.capabilities + if capabilities['browserName'] == 'firefox': + firefox_version_regex = re.compile(r'^\d+\.\d+') + version_key = 'browserVersion' if 'browserVersion' in 'capabilities' else 'version' + try: + firefox_ver = float(firefox_version_regex.search(capabilities[version_key]).group(0)) + except AttributeError: + firefox_ver = 0.0 + else: + firefox_ver = 0.0 + driver.close() + if firefox_ver < MINIMUM_FIREFOX_VERSION: + raise Exception( + 'Required firefox version not found.\n' + 'Expected: {expected_version}; Actual: {actual_version}.\n\n' + 'Make sure that the edx.devstack.firefox container is up-to-date and running\n' + '\t{expected_version}'.format( + actual_version=firefox_ver, + expected_version=MINIMUM_FIREFOX_VERSION + ) + ) + return + + # Firefox will be run as a local process expected_firefox_ver = "Mozilla Firefox " + str(MINIMUM_FIREFOX_VERSION) firefox_ver_string = subprocess.check_output("firefox --version", shell=True).strip() firefox_version_regex = re.compile(r"Mozilla Firefox (\d+.\d+)") diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index ed46c474eb..72dc2642b1 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -154,7 +154,7 @@ sqlparse>=0.2.0,<0.3.0 # Used for testing before_after==0.1.3 -bok-choy==0.7.0 +bok-choy==0.7.1 chrono==1.0.2 ddt==0.8.0 django-crum==0.5 diff --git a/scripts/reset-test-db.sh b/scripts/reset-test-db.sh index bc106e94df..b05d785cda 100755 --- a/scripts/reset-test-db.sh +++ b/scripts/reset-test-db.sh @@ -27,6 +27,14 @@ set -e DB_CACHE_DIR="common/test/db_cache" +if [[ -z "$BOK_CHOY_HOSTNAME" ]]; then + MYSQL_HOST="" + SETTINGS="bok_choy" +else + MYSQL_HOST="--host=edx.devstack.mysql" + SETTINGS="bok_choy_docker" +fi + declare -A databases declare -a database_order databases=(["default"]="edxtest" ["student_module_history"]="student_module_history_test") @@ -34,7 +42,7 @@ database_order=("default" "student_module_history") # Ensure the test database exists. for db in "${database_order[@]}"; do - echo "CREATE DATABASE IF NOT EXISTS ${databases[$db]};" | mysql -u root + echo "CREATE DATABASE IF NOT EXISTS ${databases[$db]};" | mysql $MYSQL_HOST -u root # Clear out the test database # @@ -43,7 +51,7 @@ for db in "${database_order[@]}"; do # or a jenkins worker environment) that already ran tests on another commit that had # different migrations that created, dropped, or altered tables. echo "Issuing a reset_db command to the $db bok_choy MySQL database." - ./manage.py lms --settings bok_choy reset_db --traceback --noinput --router $db + ./manage.py lms --settings $SETTINGS reset_db --traceback --noinput --router $db # If there are cached database schemas/data, load them if [[ ! -f $DB_CACHE_DIR/bok_choy_schema_$db.sql || ! -f $DB_CACHE_DIR/bok_choy_data_$db.json || ! -f $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql ]]; then @@ -61,19 +69,19 @@ if [[ -z $REBUILD_CACHE ]]; then for db in "${database_order[@]}"; do # Load the schema, then the data (including the migration history) echo "Loading the schema from the filesystem into the $db MySQL DB." - mysql -u root "${databases["$db"]}" < $DB_CACHE_DIR/bok_choy_schema_$db.sql + mysql $MYSQL_HOST -u root "${databases["$db"]}" < $DB_CACHE_DIR/bok_choy_schema_$db.sql echo "Loading the fixture data from the filesystem into the $db MySQL DB." - ./manage.py lms --settings bok_choy loaddata --database $db $DB_CACHE_DIR/bok_choy_data_$db.json + ./manage.py lms --settings $SETTINGS loaddata --database $db $DB_CACHE_DIR/bok_choy_data_$db.json # Migrations are stored in the default database echo "Loading the migration data from the filesystem into the $db MySQL DB." - mysql -u root "${databases["$db"]}" < $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql + mysql $MYSQL_HOST -u root "${databases["$db"]}" < $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql # Re-run migrations to ensure we are up-to-date echo "Running the lms migrations on the $db bok_choy DB." - ./manage.py lms --settings bok_choy migrate --database $db --traceback --noinput + ./manage.py lms --settings $SETTINGS migrate --database $db --traceback --noinput echo "Running the cms migrations on the $db bok_choy DB." - ./manage.py cms --settings bok_choy migrate --database $db --traceback --noinput + ./manage.py cms --settings $SETTINGS migrate --database $db --traceback --noinput done @@ -86,18 +94,18 @@ else for db in "${database_order[@]}"; do # Re-run migrations on the test database echo "Issuing a migrate command to the $db bok_choy MySQL database for the lms django apps." - ./manage.py lms --settings bok_choy migrate --database $db --traceback --noinput + ./manage.py lms --settings $SETTINGS migrate --database $db --traceback --noinput echo "Issuing a migrate command to the $db bok_choy MySQL database for the cms django apps." - ./manage.py cms --settings bok_choy migrate --database $db --traceback --noinput + ./manage.py cms --settings $SETTINGS migrate --database $db --traceback --noinput # Dump the schema and data to the cache echo "Using the dumpdata command to save the $db fixture data to the filesystem." - ./manage.py lms --settings bok_choy dumpdata --database $db > $DB_CACHE_DIR/bok_choy_data_$db.json --exclude=api_admin.Catalog + ./manage.py lms --settings $SETTINGS dumpdata --database $db > $DB_CACHE_DIR/bok_choy_data_$db.json --exclude=api_admin.Catalog echo "Saving the schema of the $db bok_choy DB to the filesystem." - mysqldump -u root --no-data --skip-comments --skip-dump-date "${databases[$db]}" > $DB_CACHE_DIR/bok_choy_schema_$db.sql + mysqldump $MYSQL_HOST -u root --no-data --skip-comments --skip-dump-date "${databases[$db]}" > $DB_CACHE_DIR/bok_choy_schema_$db.sql # dump_data does not dump the django_migrations table so we do it separately. echo "Saving the django_migrations table of the $db bok_choy DB to the filesystem." - mysqldump -u root --no-create-info "${databases["$db"]}" django_migrations > $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql + mysqldump $MYSQL_HOST -u root --no-create-info "${databases["$db"]}" django_migrations > $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql done fi