364 lines
13 KiB
Python
364 lines
13 KiB
Python
"""
|
|
Helper functions for loading environment settings.
|
|
"""
|
|
import configparser
|
|
import json
|
|
import os
|
|
import sys
|
|
from time import sleep
|
|
|
|
import memcache
|
|
from lazy import lazy
|
|
from path import Path as path
|
|
from paver.easy import BuildFailure, sh
|
|
|
|
from pavelib.utils.cmd import django_cmd
|
|
|
|
|
|
def repo_root():
|
|
"""
|
|
Get the root of the git repository (edx-platform).
|
|
|
|
This sometimes fails on Docker Devstack, so it's been broken
|
|
down with some additional error handling. It usually starts
|
|
working within 30 seconds or so; for more details, see
|
|
https://openedx.atlassian.net/browse/PLAT-1629 and
|
|
https://github.com/docker/for-mac/issues/1509
|
|
"""
|
|
file_path = path(__file__)
|
|
attempt = 1
|
|
while True:
|
|
try:
|
|
absolute_path = file_path.abspath()
|
|
break
|
|
except OSError:
|
|
print(f'Attempt {attempt}/180 to get an absolute path failed')
|
|
if attempt < 180:
|
|
attempt += 1
|
|
sleep(1)
|
|
else:
|
|
print('Unable to determine the absolute path of the edx-platform repo, aborting')
|
|
raise
|
|
return absolute_path.parent.parent.parent
|
|
|
|
|
|
class Env:
|
|
"""
|
|
Load information about the execution environment.
|
|
"""
|
|
|
|
# Root of the git repository (edx-platform)
|
|
REPO_ROOT = repo_root()
|
|
|
|
# Reports Directory
|
|
REPORT_DIR = REPO_ROOT / 'reports'
|
|
METRICS_DIR = REPORT_DIR / 'metrics'
|
|
QUALITY_DIR = REPORT_DIR / 'quality_junitxml'
|
|
|
|
# Generic log dir
|
|
GEN_LOG_DIR = REPO_ROOT / "test_root" / "log"
|
|
|
|
# Python unittest dirs
|
|
PYTHON_COVERAGERC = REPO_ROOT / ".coveragerc"
|
|
|
|
# Bok_choy dirs
|
|
BOK_CHOY_DIR = REPO_ROOT / "common" / "test" / "acceptance"
|
|
BOK_CHOY_LOG_DIR = GEN_LOG_DIR
|
|
BOK_CHOY_REPORT_DIR = REPORT_DIR / "bok_choy"
|
|
BOK_CHOY_A11Y_REPORT_DIR = REPORT_DIR / "a11y"
|
|
BOK_CHOY_COVERAGERC = BOK_CHOY_DIR / ".coveragerc"
|
|
BOK_CHOY_A11Y_COVERAGERC = BOK_CHOY_DIR / ".a11ycoveragerc"
|
|
BOK_CHOY_A11Y_CUSTOM_RULES_FILE = (
|
|
REPO_ROOT / "node_modules" / "edx-custom-a11y-rules" /
|
|
"lib" / "custom_a11y_rules.js"
|
|
)
|
|
|
|
# Which Python version should be used in xdist workers?
|
|
PYTHON_VERSION = os.environ.get("PYTHON_VERSION", "2.7")
|
|
|
|
# If set, put reports for run in "unique" directories.
|
|
# The main purpose of this is to ensure that the reports can be 'slurped'
|
|
# in the main jenkins flow job without overwriting the reports from other
|
|
# build steps. For local development/testing, this shouldn't be needed.
|
|
if os.environ.get("SHARD", None):
|
|
shard_str = "shard_{}".format(os.environ.get("SHARD"))
|
|
BOK_CHOY_REPORT_DIR = BOK_CHOY_REPORT_DIR / shard_str
|
|
BOK_CHOY_LOG_DIR = BOK_CHOY_LOG_DIR / shard_str
|
|
|
|
# The stubs package is currently located in the Django app called "terrain"
|
|
# from when they were used by both the bok-choy and lettuce (deprecated) acceptance tests
|
|
BOK_CHOY_STUB_DIR = REPO_ROOT / "common" / "djangoapps" / "terrain"
|
|
|
|
# Directory that videos are served from
|
|
VIDEO_SOURCE_DIR = REPO_ROOT / "test_root" / "data" / "video"
|
|
|
|
PRINT_SETTINGS_LOG_FILE = BOK_CHOY_LOG_DIR / "print_settings.log"
|
|
|
|
# Detect if in a Docker container, and if so which one
|
|
SERVER_HOST = os.environ.get('BOK_CHOY_HOSTNAME', '0.0.0.0')
|
|
USING_DOCKER = SERVER_HOST != '0.0.0.0'
|
|
SETTINGS = 'bok_choy_docker' if USING_DOCKER else 'bok_choy'
|
|
DEVSTACK_SETTINGS = 'devstack_docker' if USING_DOCKER else 'devstack'
|
|
TEST_SETTINGS = 'test'
|
|
|
|
BOK_CHOY_SERVERS = {
|
|
'lms': {
|
|
'host': SERVER_HOST,
|
|
'port': os.environ.get('BOK_CHOY_LMS_PORT', '8003'),
|
|
'log': BOK_CHOY_LOG_DIR / "bok_choy_lms.log"
|
|
},
|
|
'cms': {
|
|
'host': SERVER_HOST,
|
|
'port': os.environ.get('BOK_CHOY_CMS_PORT', '8031'),
|
|
'log': BOK_CHOY_LOG_DIR / "bok_choy_studio.log"
|
|
}
|
|
}
|
|
|
|
BOK_CHOY_STUBS = {
|
|
|
|
'xqueue': {
|
|
'port': 8040,
|
|
'log': BOK_CHOY_LOG_DIR / "bok_choy_xqueue.log",
|
|
'config': 'register_submission_url=http://0.0.0.0:8041/test/register_submission',
|
|
},
|
|
|
|
'ora': {
|
|
'port': 8041,
|
|
'log': BOK_CHOY_LOG_DIR / "bok_choy_ora.log",
|
|
'config': '',
|
|
},
|
|
|
|
'comments': {
|
|
'port': 4567,
|
|
'log': BOK_CHOY_LOG_DIR / "bok_choy_comments.log",
|
|
},
|
|
|
|
'video': {
|
|
'port': 8777,
|
|
'log': BOK_CHOY_LOG_DIR / "bok_choy_video_sources.log",
|
|
'config': f"root_dir={VIDEO_SOURCE_DIR}",
|
|
},
|
|
|
|
'youtube': {
|
|
'port': 9080,
|
|
'log': BOK_CHOY_LOG_DIR / "bok_choy_youtube.log",
|
|
},
|
|
|
|
'edxnotes': {
|
|
'port': 8042,
|
|
'log': BOK_CHOY_LOG_DIR / "bok_choy_edxnotes.log",
|
|
},
|
|
|
|
'ecommerce': {
|
|
'port': 8043,
|
|
'log': BOK_CHOY_LOG_DIR / "bok_choy_ecommerce.log",
|
|
},
|
|
|
|
'catalog': {
|
|
'port': 8091,
|
|
'log': BOK_CHOY_LOG_DIR / "bok_choy_catalog.log",
|
|
},
|
|
|
|
'lti': {
|
|
'port': 8765,
|
|
'log': BOK_CHOY_LOG_DIR / "bok_choy_lti.log",
|
|
},
|
|
}
|
|
|
|
# Mongo databases that will be dropped before/after the tests run
|
|
MONGO_HOST = 'edx.devstack.mongo' if USING_DOCKER else 'localhost'
|
|
BOK_CHOY_MONGO_DATABASE = "test"
|
|
BOK_CHOY_CACHE_HOST = 'edx.devstack.memcached' if USING_DOCKER else '0.0.0.0'
|
|
BOK_CHOY_CACHE = memcache.Client([f'{BOK_CHOY_CACHE_HOST}:11211'], debug=0)
|
|
|
|
# Test Ids Directory
|
|
TEST_DIR = REPO_ROOT / ".testids"
|
|
|
|
# Configured browser to use for the js test suites
|
|
SELENIUM_BROWSER = os.environ.get('SELENIUM_BROWSER', 'firefox')
|
|
if USING_DOCKER:
|
|
KARMA_BROWSER = 'ChromeDocker' if SELENIUM_BROWSER == 'chrome' else 'FirefoxDocker'
|
|
else:
|
|
KARMA_BROWSER = 'FirefoxNoUpdates'
|
|
|
|
# Files used to run each of the js test suites
|
|
# TODO: Store this as a dict. Order seems to matter for some
|
|
# reason. See issue TE-415.
|
|
KARMA_CONFIG_FILES = [
|
|
REPO_ROOT / 'cms/static/karma_cms.conf.js',
|
|
REPO_ROOT / 'cms/static/karma_cms_squire.conf.js',
|
|
REPO_ROOT / 'cms/static/karma_cms_webpack.conf.js',
|
|
REPO_ROOT / 'lms/static/karma_lms.conf.js',
|
|
REPO_ROOT / 'common/lib/xmodule/xmodule/js/karma_xmodule.conf.js',
|
|
REPO_ROOT / 'common/lib/xmodule/xmodule/js/karma_xmodule_webpack.conf.js',
|
|
REPO_ROOT / 'common/static/karma_common.conf.js',
|
|
REPO_ROOT / 'common/static/karma_common_requirejs.conf.js',
|
|
]
|
|
|
|
JS_TEST_ID_KEYS = [
|
|
'cms',
|
|
'cms-squire',
|
|
'cms-webpack',
|
|
'lms',
|
|
'xmodule',
|
|
'xmodule-webpack',
|
|
'common',
|
|
'common-requirejs',
|
|
'jest-snapshot'
|
|
]
|
|
|
|
JS_REPORT_DIR = REPORT_DIR / 'javascript'
|
|
|
|
# Directories used for common/lib/tests
|
|
IGNORED_TEST_DIRS = ('__pycache__', '.cache', '.pytest_cache')
|
|
LIB_TEST_DIRS = []
|
|
for item in (REPO_ROOT / "common/lib").listdir():
|
|
dir_name = (REPO_ROOT / 'common/lib' / item)
|
|
if dir_name.isdir() and not dir_name.endswith(IGNORED_TEST_DIRS):
|
|
LIB_TEST_DIRS.append(path("common/lib") / item.basename())
|
|
LIB_TEST_DIRS.append(path("pavelib/paver_tests"))
|
|
LIB_TEST_DIRS.append(path("scripts/xsslint/tests"))
|
|
|
|
# Directory for i18n test reports
|
|
I18N_REPORT_DIR = REPORT_DIR / 'i18n'
|
|
|
|
# Service variant (lms, cms, etc.) configured with an environment variable
|
|
# We use this to determine which envs.json file to load.
|
|
SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None)
|
|
|
|
# If service variant not configured in env, then pass the correct
|
|
# environment for lms / cms
|
|
if not SERVICE_VARIANT: # this will intentionally catch "";
|
|
if any(i in sys.argv[1:] for i in ('cms', 'studio')):
|
|
SERVICE_VARIANT = 'cms'
|
|
else:
|
|
SERVICE_VARIANT = 'lms'
|
|
|
|
@classmethod
|
|
def get_django_settings(cls, django_settings, system, settings=None, print_setting_args=None):
|
|
"""
|
|
Interrogate Django environment for specific settings values
|
|
:param django_settings: list of django settings values to get
|
|
:param system: the django app to use when asking for the setting (lms | cms)
|
|
:param settings: the settings file to use when asking for the value
|
|
:param print_setting_args: the additional arguments to send to print_settings
|
|
:return: unicode value of the django setting
|
|
"""
|
|
if not settings:
|
|
settings = os.environ.get("EDX_PLATFORM_SETTINGS", "aws")
|
|
log_dir = os.path.dirname(cls.PRINT_SETTINGS_LOG_FILE)
|
|
if not os.path.exists(log_dir):
|
|
os.makedirs(log_dir)
|
|
settings_length = len(django_settings)
|
|
django_settings = ' '.join(django_settings) # parse_known_args makes a list again
|
|
print_setting_args = ' '.join(print_setting_args or [])
|
|
try:
|
|
value = sh(
|
|
django_cmd(
|
|
system,
|
|
settings,
|
|
"print_setting {django_settings} 2>{log_file} {print_setting_args}".format(
|
|
django_settings=django_settings,
|
|
print_setting_args=print_setting_args,
|
|
log_file=cls.PRINT_SETTINGS_LOG_FILE
|
|
).strip()
|
|
),
|
|
capture=True
|
|
)
|
|
# else for cases where values are not found & sh returns one None value
|
|
return tuple(str(value).splitlines()) if value else tuple(None for _ in range(settings_length))
|
|
except BuildFailure:
|
|
print(f"Unable to print the value of the {django_settings} setting:")
|
|
with open(cls.PRINT_SETTINGS_LOG_FILE) as f:
|
|
print(f.read())
|
|
sys.exit(1)
|
|
|
|
@classmethod
|
|
def get_django_json_settings(cls, django_settings, system, settings=None):
|
|
"""
|
|
Interrogate Django environment for specific settings value
|
|
:param django_settings: list of django settings values to get
|
|
:param system: the django app to use when asking for the setting (lms | cms)
|
|
:param settings: the settings file to use when asking for the value
|
|
:return: json string value of the django setting
|
|
"""
|
|
return cls.get_django_settings(
|
|
django_settings,
|
|
system,
|
|
settings=settings,
|
|
print_setting_args=["--json"],
|
|
)
|
|
|
|
@classmethod
|
|
def covered_modules(cls):
|
|
"""
|
|
List the source modules listed in .coveragerc for which coverage
|
|
will be measured.
|
|
"""
|
|
coveragerc = configparser.RawConfigParser()
|
|
coveragerc.read(cls.PYTHON_COVERAGERC)
|
|
modules = coveragerc.get('run', 'source')
|
|
result = []
|
|
for module in modules.split('\n'):
|
|
module = module.strip()
|
|
if module:
|
|
result.append(module)
|
|
return result
|
|
|
|
@lazy
|
|
def env_tokens(self):
|
|
"""
|
|
Return a dict of environment settings.
|
|
If we couldn't find the JSON file, issue a warning and return an empty dict.
|
|
"""
|
|
|
|
# Find the env JSON file
|
|
if self.SERVICE_VARIANT:
|
|
env_path = self.REPO_ROOT.parent / f"{self.SERVICE_VARIANT}.env.json"
|
|
else:
|
|
env_path = path("env.json").abspath()
|
|
|
|
# If the file does not exist, here or one level up,
|
|
# issue a warning and return an empty dict
|
|
if not env_path.isfile():
|
|
env_path = env_path.parent.parent / env_path.basename()
|
|
if not env_path.isfile():
|
|
print(
|
|
"Warning: could not find environment JSON file "
|
|
"at '{path}'".format(path=env_path),
|
|
file=sys.stderr,
|
|
)
|
|
return dict()
|
|
|
|
# Otherwise, load the file as JSON and return the resulting dict
|
|
try:
|
|
with open(env_path) as env_file:
|
|
return json.load(env_file)
|
|
|
|
except ValueError:
|
|
print(
|
|
"Error: Could not parse JSON "
|
|
"in {path}".format(path=env_path),
|
|
file=sys.stderr,
|
|
)
|
|
sys.exit(1)
|
|
|
|
@lazy
|
|
def feature_flags(self):
|
|
"""
|
|
Return a dictionary of feature flags configured by the environment.
|
|
"""
|
|
return self.env_tokens.get('FEATURES', dict())
|
|
|
|
@classmethod
|
|
def rsync_dirs(cls):
|
|
"""
|
|
List the directories that should be synced during pytest-xdist
|
|
execution. Needs to include all modules for which coverage is
|
|
measured, not just the tests being run.
|
|
"""
|
|
result = set()
|
|
for module in cls.covered_modules():
|
|
result.add(module.split('/')[0])
|
|
return result
|