Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix unicode strings in pavelib/

Browse Source
This commit is contained in:
Matthew Piatetsky
2019-02-01 16:44:26 -05:00
parent bf692acea6
commit 6cf44a0467
29 changed files with 204 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pavelib/acceptance_test.py
View File

@@ -61,5 +61,5 @@ def test_acceptance(options, passthrough_options):
)
print(msg)
suite = AcceptanceTestSuite('{} acceptance'.format(opts['system']), **opts)
suite = AcceptanceTestSuite(u'{} acceptance'.format(opts['system']), **opts)
suite.run()

46
pavelib/assets.py
View File

@@ -89,7 +89,7 @@ SASS_LOOKUP_DEPENDENCIES = {
COLLECTSTATIC_LOG_DIR_ARG = 'collect_log_dir'
# Webpack command
WEBPACK_COMMAND = 'STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} $(npm bin)/webpack {options}'
WEBPACK_COMMAND = u'STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} $(npm bin)/webpack {options}'
def get_sass_directories(system, theme_dir=None):
@@ -110,7 +110,7 @@ def get_sass_directories(system, theme_dir=None):
:param theme_dir: absolute path of theme for which to compile sass files.
"""
if system not in SYSTEMS:
raise ValueError("'system' must be one of ({allowed_values})".format(allowed_values=', '.join(SYSTEMS.keys())))
raise ValueError(u"'system' must be one of ({allowed_values})".format(allowed_values=', '.join(SYSTEMS.keys())))
system = SYSTEMS[system]
applicable_directories = list()
@@ -453,7 +453,7 @@ def compile_sass(options):
for system in systems:
for theme in themes:
print("Started compiling '{system}' Sass for '{theme}'.".format(system=system, theme=theme or 'system'))
print(u"Started compiling '{system}' Sass for '{theme}'.".format(system=system, theme=theme or 'system'))
# Compile sass files
is_successful = _compile_sass(
@@ -465,18 +465,18 @@ def compile_sass(options):
)
if is_successful:
print("Finished compiling '{system}' Sass for '{theme}'.".format(
print(u"Finished compiling '{system}' Sass for '{theme}'.".format(
system=system, theme=theme or 'system'
))
compilation_results['success' if is_successful else 'failure'].append('{system} sass for {theme}.'.format(
compilation_results['success' if is_successful else 'failure'].append(u'{system} sass for {theme}.'.format(
system=system, theme=theme or 'system',
))
print("\t\tFinished compiling Sass:")
if not dry_run:
for sass_dir, css_dir, duration in timing_info:
print(">> {} -> {} in {}s".format(sass_dir, css_dir, duration))
print(u">> {} -> {} in {}s".format(sass_dir, css_dir, duration))
if compilation_results['success']:
print("\033[92m\nSuccessful compilations:\n--- " + "\n--- ".join(compilation_results['success']) + "\n\033[00m")
@@ -522,7 +522,7 @@ def _compile_sass(system, theme, debug, force, timing_info):
lookup_paths = dirs['lookup_paths']
if not sass_source_dir.isdir():
print("\033[91m Sass dir '{dir}' does not exists, skipping sass compilation for '{theme}' \033[00m".format(
print(u"\033[91m Sass dir '{dir}' does not exists, skipping sass compilation for '{theme}' \033[00m".format(
dir=sass_dirs, theme=theme or system,
))
# theme doesn't override sass directory, so skip it
@@ -530,14 +530,14 @@ def _compile_sass(system, theme, debug, force, timing_info):
if force:
if dry_run:
tasks.environment.info("rm -rf {css_dir}/*.css".format(
tasks.environment.info(u"rm -rf {css_dir}/*.css".format(
css_dir=css_dir,
))
else:
sh("rm -rf {css_dir}/*.css".format(css_dir=css_dir))
sh(u"rm -rf {css_dir}/*.css".format(css_dir=css_dir))
if dry_run:
tasks.environment.info("libsass {sass_dir}".format(
tasks.environment.info(u"libsass {sass_dir}".format(
sass_dir=sass_source_dir,
))
else:
@@ -554,7 +554,7 @@ def _compile_sass(system, theme, debug, force, timing_info):
if should_generate_rtl_css_file(sass_file):
source_css_file = sass_file.replace(sass_source_dir, css_dir).replace('.scss', '.css')
target_css_file = source_css_file.replace('.css', '-rtl.css')
sh("rtlcss {source_file} {target_file}".format(
sh(u"rtlcss {source_file} {target_file}".format(
source_file=source_css_file,
target_file=target_css_file,
))
@@ -604,19 +604,19 @@ def process_npm_assets():
else:
vendor_dir = NPM_JS_VENDOR_DIRECTORY
if os.path.exists(library_path):
sh('/bin/cp -rf {library_path} {vendor_dir}'.format(
sh(u'/bin/cp -rf {library_path} {vendor_dir}'.format(
library_path=library_path,
vendor_dir=vendor_dir,
))
elif not skip_if_missing:
raise Exception('Missing vendor file {library_path}'.format(library_path=library_path))
raise Exception(u'Missing vendor file {library_path}'.format(library_path=library_path))
def copy_vendor_library_dir(library_dir, skip_if_missing=False):
"""
Copies all vendor libraries in directory to the shared vendor directory.
"""
library_dir_path = 'node_modules/{library_dir}'.format(library_dir=library_dir)
print('Copying vendor library dir: {}'.format(library_dir_path))
print(u'Copying vendor library dir: {}'.format(library_dir_path))
if os.path.exists(library_dir_path):
for dirpath, _, filenames in os.walk(library_dir_path):
for filename in filenames:
@@ -695,16 +695,16 @@ def collect_assets(systems, settings, **kwargs):
]
ignore_args = " ".join(
'--ignore "{}"'.format(pattern) for pattern in ignore_patterns
u'--ignore "{}"'.format(pattern) for pattern in ignore_patterns
)
for sys in systems:
collectstatic_stdout_str = _collect_assets_cmd(sys, **kwargs)
sh(django_cmd(sys, settings, "collectstatic {ignore_args} --noinput {logfile_str}".format(
sh(django_cmd(sys, settings, u"collectstatic {ignore_args} --noinput {logfile_str}".format(
ignore_args=ignore_args,
logfile_str=collectstatic_stdout_str
)))
print("\t\tFinished collecting {} assets.".format(sys))
print(u"\t\tFinished collecting {} assets.".format(sys))
def _collect_assets_cmd(system, **kwargs):
@@ -717,7 +717,7 @@ def _collect_assets_cmd(system, **kwargs):
if kwargs[COLLECTSTATIC_LOG_DIR_ARG] is None:
collectstatic_stdout_str = ""
else:
collectstatic_stdout_str = "> {output_dir}/{sys}-collectstatic.log".format(
collectstatic_stdout_str = u"> {output_dir}/{sys}-collectstatic.log".format(
output_dir=kwargs[COLLECTSTATIC_LOG_DIR_ARG],
sys=system
)
@@ -743,7 +743,7 @@ def execute_compile_sass(args):
django_cmd(
sys,
args.settings,
"compile_sass {system} {options}".format(
u"compile_sass {system} {options}".format(
system='cms' if sys == 'studio' else sys,
options=options,
),
@@ -765,14 +765,14 @@ def webpack(options):
static_root_lms = Env.get_django_setting("STATIC_ROOT", "lms", settings=settings)
static_root_cms = Env.get_django_setting("STATIC_ROOT", "cms", settings=settings)
config_path = Env.get_django_setting("WEBPACK_CONFIG_PATH", "lms", settings=settings)
environment = 'NODE_ENV={node_env} STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms}'.format(
environment = u'NODE_ENV={node_env} STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms}'.format(
node_env="development" if config_path == 'webpack.dev.config.js' else "production",
static_root_lms=static_root_lms,
static_root_cms=static_root_cms
)
sh(
cmd(
'{environment} $(npm bin)/webpack --config={config_path}'.format(
u'{environment} $(npm bin)/webpack --config={config_path}'.format(
environment=environment,
config_path=config_path
)
@@ -788,8 +788,8 @@ def execute_webpack_watch(settings=None):
# not all JS files, so we use its own watcher instead of subclassing
# from Watchdog like the other watchers do.
run_background_process(
'STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} $(npm bin)/webpack {options}'.format(
options='--watch --config={config_path}'.format(
u'STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} $(npm bin)/webpack {options}'.format(
options=u'--watch --config={config_path}'.format(
config_path=Env.get_django_setting("WEBPACK_CONFIG_PATH", "lms", settings=settings)
),
static_root_lms=Env.get_django_setting("STATIC_ROOT", "lms", settings=settings),

10
pavelib/bok_choy.py
View File

@@ -143,7 +143,7 @@ def run_bokchoy(options, passthrough_options):
test_suite = BokChoyTestSuite('bok-choy', passthrough_options=passthrough_options, **options)
msg = colorize(
'green',
'Running tests using {default_store} modulestore.'.format(
u'Running tests using {default_store} modulestore.'.format(
default_store=test_suite.default_store,
)
)
@@ -160,14 +160,14 @@ def parse_coverage(report_dir, coveragerc):
msg = colorize('green', "Combining coverage reports")
print(msg)
sh("coverage combine --rcfile={}".format(coveragerc))
sh(u"coverage combine --rcfile={}".format(coveragerc))
msg = colorize('green', "Generating coverage reports")
print(msg)
sh("coverage html --rcfile={}".format(coveragerc))
sh("coverage xml --rcfile={}".format(coveragerc))
sh("coverage report --rcfile={}".format(coveragerc))
sh(u"coverage html --rcfile={}".format(coveragerc))
sh(u"coverage xml --rcfile={}".format(coveragerc))
sh(u"coverage report --rcfile={}".format(coveragerc))
@task

4
pavelib/database.py
View File

@@ -103,7 +103,7 @@ def update_local_bokchoy_db_from_s3(options):
reset_test_db(BOKCHOY_DB_FILES, update_cache_files=False, use_existing_db=True)
else:
msg = "{} {} {}".format(
msg = u"{} {} {}".format(
"Did not find updated bokchoy db files at S3.",
"Loading the bokchoy db files from disk",
"and running migrations."
@@ -116,7 +116,7 @@ def update_local_bokchoy_db_from_s3(options):
if not is_fingerprint_in_bucket(fingerprint, CACHE_BUCKET_NAME):
upload_db_cache_to_s3(fingerprint, BOKCHOY_DB_FILES, CACHE_BUCKET_NAME)
else:
msg = "{} {}. {}".format(
msg = u"{} {}. {}".format(
"Found a matching fingerprint in bucket ",
CACHE_BUCKET_NAME,
"Not pushing to s3"

10
pavelib/docs.py
View File

@@ -40,8 +40,8 @@ def doc_path(options, allow_default=True):
if doc_type == 'default' and not allow_default:
print(
"You must specify a documentation type using '--type'. "
"Valid options are: {options}".format(
u"You must specify a documentation type using '--type'. "
u"Valid options are: {options}".format(
options=valid_doc_types()
)
)
@@ -49,8 +49,8 @@ def doc_path(options, allow_default=True):
if path is None:
print(
"Invalid documentation type '{doc_type}'. "
"Valid options are: {options}".format(
u"Invalid documentation type '{doc_type}'. "
u"Valid options are: {options}".format(
doc_type=doc_type, options=valid_doc_types()
)
)
@@ -73,7 +73,7 @@ def build_docs(options):
"""
verbose = getattr(options, 'verbose', False)
cmd = "cd {dir}; make html quiet={quiet}".format(
cmd = u"cd {dir}; make html quiet={quiet}".format(
dir=doc_path(options),
quiet="false" if verbose else "true"
)

6
pavelib/i18n.py
View File

@@ -131,9 +131,9 @@ def i18n_validate_transifex_config():
if not config.isfile or config.getsize == 0:
msg = colorize(
'red',
"Cannot connect to Transifex, config file is missing"
" or empty: {config} \nSee "
"http://help.transifex.com/features/client/#transifexrc \n".format(
u"Cannot connect to Transifex, config file is missing"
u" or empty: {config} \nSee "
u"http://help.transifex.com/features/client/#transifexrc \n".format(
config=config,
)
)

2
pavelib/js_test.py
View File

@@ -48,7 +48,7 @@ def test_js(options):
if (suite != 'all') and (suite not in Env.JS_TEST_ID_KEYS):
sys.stderr.write(
"Unknown test suite. Please choose from ({suites})\n".format(
u"Unknown test suite. Please choose from ({suites})\n".format(
suites=", ".join(Env.JS_TEST_ID_KEYS)
)
)

6
pavelib/paver_tests/test_assets.py
View File

@@ -147,7 +147,7 @@ class TestPaverThemeAssetTasks(PaverTestCase):
)
if force:
expected_messages.append(
'rm -rf {test_theme_dir}/lms/static/css/*.css'.format(test_theme_dir=str(TEST_THEME_DIR))
u'rm -rf {test_theme_dir}/lms/static/css/*.css'.format(test_theme_dir=str(TEST_THEME_DIR))
)
expected_messages.append(
u'libsass {test_theme_dir}/lms/static/sass'.format(test_theme_dir=str(TEST_THEME_DIR))
@@ -345,7 +345,7 @@ class TestCollectAssets(PaverTestCase):
"""
for i, sys in enumerate(systems):
msg = self.task_messages[i]
self.assertTrue(msg.startswith('python manage.py {}'.format(sys)))
self.assertTrue(msg.startswith(u'python manage.py {}'.format(sys)))
self.assertIn(' collectstatic ', msg)
self.assertIn('--settings={}'.format(Env.DEVSTACK_SETTINGS), msg)
self.assertTrue(msg.endswith(' {}'.format(log_location)))
@@ -372,7 +372,7 @@ class TestUpdateAssetsTask(PaverTestCase):
call_task('pavelib.assets.update_assets', args=cmd_args)
self.assertTrue(
self._is_substring_in_list(self.task_messages, expected_substring),
msg="{substring} not found in messages".format(substring=expected_substring)
msg=u"{substring} not found in messages".format(substring=expected_substring)
)
def _is_substring_in_list(self, messages_list, expected_substring):

12
pavelib/paver_tests/test_database.py
View File

@@ -115,8 +115,8 @@ class TestPaverDatabaseTasks(MockS3Mixin, PaverTestCase):
# Make sure that the local cache files are used - NOT downloaded from s3
self.assertFalse(_mock_get_file.called)
calls = [
call('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)),
call('{}/scripts/reset-test-db.sh --use-existing-db'.format(Env.REPO_ROOT))
call(u'{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)),
call(u'{}/scripts/reset-test-db.sh --use-existing-db'.format(Env.REPO_ROOT))
]
_mock_sh.assert_has_calls(calls)
@@ -156,8 +156,8 @@ class TestPaverDatabaseTasks(MockS3Mixin, PaverTestCase):
'moto_test_bucket', self.fingerprint_filename, db_utils.CACHE_FOLDER
)
calls = [
call('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)),
call('{}/scripts/reset-test-db.sh --use-existing-db'.format(Env.REPO_ROOT))
call(u'{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)),
call(u'{}/scripts/reset-test-db.sh --use-existing-db'.format(Env.REPO_ROOT))
]
_mock_sh.assert_has_calls(calls)
@@ -184,8 +184,8 @@ class TestPaverDatabaseTasks(MockS3Mixin, PaverTestCase):
database.update_local_bokchoy_db_from_s3() # pylint: disable=no-value-for-parameter
calls = [
call('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)),
call('{}/scripts/reset-test-db.sh --rebuild_cache --use-existing-db'.format(Env.REPO_ROOT))
call(u'{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)),
call(u'{}/scripts/reset-test-db.sh --rebuild_cache --use-existing-db'.format(Env.REPO_ROOT))
]
_mock_sh.assert_has_calls(calls)

6
pavelib/paver_tests/test_extract_and_generate.py
View File

@@ -84,11 +84,11 @@ class TestGenerate(TestCase):
mofile = filename + '.mo'
path = os.path.join(self.configuration.get_messages_dir(locale), mofile)
exists = os.path.exists(path)
self.assertTrue(exists, msg='Missing file in locale %s: %s' % (locale, mofile))
self.assertTrue(exists, msg=u'Missing file in locale %s: %s' % (locale, mofile))
self.assertGreaterEqual(
datetime.fromtimestamp(os.path.getmtime(path), UTC),
self.start_time,
msg='File not recently modified: %s' % path
msg=u'File not recently modified: %s' % path
)
# Segmenting means that the merge headers don't work they way they
# used to, so don't make this check for now. I'm not sure if we'll
@@ -113,7 +113,7 @@ class TestGenerate(TestCase):
self.assertEqual(
len(match),
3,
msg="Found %s (should be 3) merge comments in the header for %s" % (len(match), path)
msg=u"Found %s (should be 3) merge comments in the header for %s" % (len(match), path)
)

8
pavelib/paver_tests/test_paver_bok_choy_cmds.py
View File

@@ -145,7 +145,7 @@ class TestPaverBokChoyCmd(unittest.TestCase):
repo_dir=REPO_DIR,
shard_str='/shard_' + self.shard if self.shard else '',
),
"-n {}".format(process_count),
u"-n {}".format(process_count),
"--color=no",
"--verbose",
]
@@ -162,7 +162,7 @@ class TestPaverBokChoyCmd(unittest.TestCase):
repo_dir=REPO_DIR,
shard_str='/shard_' + self.shard if self.shard else '',
),
"-n {}".format(process_count),
u"-n {}".format(process_count),
"--color=no",
"--verbose",
]
@@ -244,9 +244,9 @@ class TestPaverPa11yCrawlerCmd(unittest.TestCase):
else:
self._mock_sh.assert_has_calls([
call(
'wget {targz} -O {dir}demo_course.tar.gz'.format(targz=DEMO_COURSE_TAR_GZ, dir=downloaded_to)),
u'wget {targz} -O {dir}demo_course.tar.gz'.format(targz=DEMO_COURSE_TAR_GZ, dir=downloaded_to)),
call(
'tar zxf {dir}demo_course.tar.gz -C {dir}'.format(dir=downloaded_to)),
u'tar zxf {dir}demo_course.tar.gz -C {dir}'.format(dir=downloaded_to)),
])
@patch("pavelib.utils.test.suites.bokchoy_suite.path")

11
pavelib/paver_tests/test_paver_pytest_cmds.py
View File

@@ -1,3 +1,4 @@
# pylint: disable=unicode-format-string
"""
Tests for the pytest paver commands themselves.
Run just this test with: paver test_lib -t pavelib/paver_tests/test_paver_pytest_cmds.py
@@ -47,26 +48,26 @@ class TestPaverPytestCmd(unittest.TestCase):
processes = 1
if pytestSubclass == "SystemTestSuite":
django_env_var_cmd = "export DJANGO_SETTINGS_MODULE={}.envs.test".format(root)
django_env_var_cmd = u"export DJANGO_SETTINGS_MODULE={}.envs.test".format(root)
elif pytestSubclass == "LibTestSuite":
if 'pavelib/paver_tests' in test_id:
django_env_var_cmd = "export DJANGO_SETTINGS_MODULE={}.envs.test".format(root)
django_env_var_cmd = u"export DJANGO_SETTINGS_MODULE={}.envs.test".format(root)
else:
django_env_var_cmd = "export DJANGO_SETTINGS_MODULE='openedx.tests.settings'"
xdist_string = '--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \
xdist_string = u'--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \
'//python="source /edx/app/edxapp/edxapp_env; {}; python"' \
'//chdir="/edx/app/edxapp/edx-platform"' \
.format(processes, ip, django_env_var_cmd)
expected_statement.append(xdist_string)
for rsync_dir in Env.rsync_dirs():
expected_statement.append('--rsyncdir {}'.format(rsync_dir))
expected_statement.append(u'--rsyncdir {}'.format(rsync_dir))
else:
if processes == -1:
expected_statement.append('-n auto')
expected_statement.append('--dist=loadscope')
elif processes != 0:
expected_statement.append('-n {}'.format(processes))
expected_statement.append(u'-n {}'.format(processes))
expected_statement.append('--dist=loadscope')
expected_statement.extend([

2
pavelib/paver_tests/test_prereqs.py
View File

@@ -30,7 +30,7 @@ class TestPaverPrereqInstall(unittest.TestCase):
self.assertEqual(
pavelib.prereqs.no_prereq_install(),
expected_val,
'NO_PREREQ_INSTALL is set to {}, but we read it as {}'.format(
u'NO_PREREQ_INSTALL is set to {}, but we read it as {}'.format(
set_val, expected_val),
)

8
pavelib/paver_tests/test_servers.py
View File

@@ -48,7 +48,7 @@ EXPECTED_PRINT_SETTINGS_COMMAND = [
]
EXPECTED_WEBPACK_COMMAND = (
u"NODE_ENV={node_env} STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} "
"$(npm bin)/webpack --config={webpack_config_path}"
u"$(npm bin)/webpack --config={webpack_config_path}"
)
@@ -167,7 +167,7 @@ class TestPaverServerTasks(PaverTestCase):
settings = options.get("settings", Env.DEVSTACK_SETTINGS)
call_task("pavelib.servers.update_db", options=options)
# pylint: disable=line-too-long
db_command = "NO_EDXAPP_SUDO=1 EDX_PLATFORM_SETTINGS_OVERRIDE={settings} /edx/bin/edxapp-migrate-{server} --traceback --pythonpath=. "
db_command = u"NO_EDXAPP_SUDO=1 EDX_PLATFORM_SETTINGS_OVERRIDE={settings} /edx/bin/edxapp-migrate-{server} --traceback --pythonpath=. "
self.assertEquals(
self.task_messages,
[
@@ -192,8 +192,8 @@ class TestPaverServerTasks(PaverTestCase):
self.assertEquals(
self.task_messages,
[
"echo 'import {system}.envs.{settings}' "
"| python manage.py {system} --settings={settings} shell --plain --pythonpath=.".format(
u"echo 'import {system}.envs.{settings}' "
u"| python manage.py {system} --settings={settings} shell --plain --pythonpath=.".format(
system=system, settings=settings
),
]

4
pavelib/paver_tests/test_utils.py
View File

@@ -19,7 +19,7 @@ class TestUtils(unittest.TestCase):
@patch('subprocess.check_output')
def test_firefox_version_ok(self, _mock_subprocesss):
test_version = MINIMUM_FIREFOX_VERSION
_mock_subprocesss.return_value = "Mozilla Firefox {version}".format(
_mock_subprocesss.return_value = u"Mozilla Firefox {version}".format(
version=str(test_version)
)
# No exception should be raised
@@ -28,7 +28,7 @@ class TestUtils(unittest.TestCase):
@patch('subprocess.check_output')
def test_firefox_version_below_expected(self, _mock_subprocesss):
test_version = MINIMUM_FIREFOX_VERSION - 1
_mock_subprocesss.return_value = "Mozilla Firefox {version}".format(
_mock_subprocesss.return_value = u"Mozilla Firefox {version}".format(
version=test_version
)
with self.assertRaises(Exception):

10
pavelib/prereqs.py
View File

@@ -121,7 +121,7 @@ def prereq_cache(cache_name, paths, install_func):
post_install_hash = compute_fingerprint(paths)
cache_file.write(post_install_hash)
else:
print('{cache} unchanged, skipping...'.format(cache=cache_name))
print(u'{cache} unchanged, skipping...'.format(cache=cache_name))
def node_prereqs_installation():
@@ -157,7 +157,7 @@ def node_prereqs_installation():
proc.wait()
else:
raise BuildFailure(error_text)
print("Successfully installed NPM packages. Log found at {}".format(
print(u"Successfully installed NPM packages. Log found at {}".format(
npm_log_file_path
))
@@ -173,7 +173,7 @@ def python_prereqs_installation():
def pip_install_req_file(req_file):
"""Pip install the requirements file."""
pip_cmd = 'pip install -q --disable-pip-version-check --exists-action w'
sh("{pip_cmd} -r {req_file}".format(pip_cmd=pip_cmd, req_file=req_file))
sh(u"{pip_cmd} -r {req_file}".format(pip_cmd=pip_cmd, req_file=req_file))
@task
@@ -243,7 +243,7 @@ def uninstall_python_packages():
for package_name in PACKAGES_TO_UNINSTALL:
if package_in_frozen(package_name, frozen):
# Uninstall the pacakge
sh("pip uninstall --disable-pip-version-check -y {}".format(package_name))
sh(u"pip uninstall --disable-pip-version-check -y {}".format(package_name))
uninstalled = True
if not uninstalled:
break
@@ -337,7 +337,7 @@ def install_prereqs():
def log_installed_python_prereqs():
""" Logs output of pip freeze for debugging. """
sh("pip freeze > {}".format(Env.GEN_LOG_DIR + "/pip_freeze.log"))
sh(u"pip freeze > {}".format(Env.GEN_LOG_DIR + "/pip_freeze.log"))
return

79
pavelib/quality.py
View File

@@ -1,3 +1,4 @@
# pylint: disable=unicode-format-string
# coding=utf-8
"""
@@ -18,12 +19,12 @@ from .utils.envs import Env
from .utils.timer import timed
ALL_SYSTEMS = 'lms,cms,common,openedx,pavelib'
JUNIT_XML_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
JUNIT_XML_TEMPLATE = u"""<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="{name}" tests="1" errors="0" failures="{failure_count}" skip="0">
<testcase classname="pavelib.quality" name="{name}" time="{seconds}">{failure_element}</testcase>
</testsuite>
"""
JUNIT_XML_FAILURE_TEMPLATE = '<failure message={message}/>'
JUNIT_XML_FAILURE_TEMPLATE = u'<failure message={message}/>'
START_TIME = datetime.utcnow()
@@ -97,7 +98,7 @@ def find_fixme(options):
apps_list = ' '.join(top_python_dirs(system))
cmd = (
"pylint --disable all --enable=fixme "
u"pylint --disable all --enable=fixme "
"--output-format=parseable {apps} "
"> {report_dir}/pylint_fixme.report".format(
apps=apps_list,
@@ -139,7 +140,7 @@ def _get_pylint_violations(systems=ALL_SYSTEMS.split(','), errors_only=False, cl
system_report = report_dir / 'pylint.report'
if clean or not system_report.exists():
sh(
"pylint {flags} --output-format=parseable {apps} "
u"pylint {flags} --output-format=parseable {apps} "
"> {report_dir}/pylint.report".format(
flags=" ".join(flags),
apps=apps_list,
@@ -190,7 +191,7 @@ def run_pylint(options):
if num_violations < lower_violations_limit > -1:
fail_quality(
result_name,
"FAILURE: Too few pylint violations. "
u"FAILURE: Too few pylint violations. "
"Expected to see at least {lower_limit} pylint violations. "
"Either pylint is not running correctly -or- "
"the limits should be lowered and/or the lower limit should be removed.".format(
@@ -202,7 +203,7 @@ def run_pylint(options):
if num_violations > upper_violations_limit > -1:
fail_quality(
result_name,
"FAILURE: Too many pylint violations. "
u"FAILURE: Too many pylint violations. "
"The limit is {upper_limit}.".format(upper_limit=upper_violations_limit)
)
else:
@@ -264,7 +265,7 @@ def _get_pep8_violations(clean=True):
Env.METRICS_DIR.makedirs_p()
if not report.exists():
sh('pycodestyle . | tee {} -a'.format(report))
sh(u'pycodestyle . | tee {} -a'.format(report))
violations_list = _pep8_violations(report)
@@ -294,7 +295,7 @@ def run_pep8(options): # pylint: disable=unused-argument
violations_list = ''.join(violations_list)
# Print number of violations to log
violations_count_str = "Number of PEP 8 violations: {count}".format(count=count)
violations_count_str = u"Number of PEP 8 violations: {count}".format(count=count)
print(violations_count_str)
print(violations_list)
@@ -331,7 +332,7 @@ def run_complexity():
print("--> Calculating cyclomatic complexity of python files...")
try:
sh(
"radon cc {system_string} --total-average > {complexity_report}".format(
u"radon cc {system_string} --total-average > {complexity_report}".format(
system_string=system_string,
complexity_report=complexity_report
)
@@ -342,7 +343,7 @@ def run_complexity():
(Env.METRICS_DIR / "python_complexity")
)
print("--> Python cyclomatic complexity report complete.")
print("radon cyclomatic complexity score: {metric}".format(metric=str(complexity_metric)))
print(u"radon cyclomatic complexity score: {metric}".format(metric=str(complexity_metric)))
except BuildFailure:
print("FAILURE: Unable to calculate python-only code-complexity.")
@@ -369,7 +370,7 @@ def run_eslint(options):
violations_limit = int(getattr(options, 'limit', -1))
sh(
"nodejs --max_old_space_size=4096 node_modules/.bin/eslint "
u"nodejs --max_old_space_size=4096 node_modules/.bin/eslint "
"--ext .js --ext .jsx --format=compact . | tee {eslint_report}".format(
eslint_report=eslint_report
),
@@ -381,7 +382,7 @@ def run_eslint(options):
except TypeError:
fail_quality(
'eslint',
"FAILURE: Number of eslint violations could not be found in {eslint_report}".format(
u"FAILURE: Number of eslint violations could not be found in {eslint_report}".format(
eslint_report=eslint_report
)
)
@@ -393,7 +394,7 @@ def run_eslint(options):
if num_violations > violations_limit > -1:
fail_quality(
'eslint',
"FAILURE: Too many eslint violations ({count}).\nThe limit is {violations_limit}.".format(
u"FAILURE: Too many eslint violations ({count}).\nThe limit is {violations_limit}.".format(
count=num_violations, violations_limit=violations_limit
)
)
@@ -411,7 +412,7 @@ def _get_stylelint_violations():
formatter = 'node_modules/stylelint-formatter-pretty'
sh(
"stylelint **/*.scss --custom-formatter={formatter} | tee {stylelint_report}".format(
u"stylelint **/*.scss --custom-formatter={formatter} | tee {stylelint_report}".format(
formatter=formatter,
stylelint_report=stylelint_report,
),
@@ -423,7 +424,7 @@ def _get_stylelint_violations():
except TypeError:
fail_quality(
'stylelint',
"FAILURE: Number of stylelint violations could not be found in {stylelint_report}".format(
u"FAILURE: Number of stylelint violations could not be found in {stylelint_report}".format(
stylelint_report=stylelint_report
)
)
@@ -450,7 +451,7 @@ def run_stylelint(options):
if num_violations > violations_limit > -1:
fail_quality(
'stylelint',
"FAILURE: Stylelint failed with too many violations: ({count}).\nThe limit is {violations_limit}.".format(
u"FAILURE: Stylelint failed with too many violations: ({count}).\nThe limit is {violations_limit}.".format(
count=num_violations,
violations_limit=violations_limit,
)
@@ -493,7 +494,7 @@ def run_xsslint(options):
_prepare_report_dir(xsslint_report_dir)
sh(
"{repo_root}/scripts/xsslint/{xsslint_script} --rule-totals --config={cfg_module} >> {xsslint_report}".format(
u"{repo_root}/scripts/xsslint/{xsslint_script} --rule-totals --config={cfg_module} >> {xsslint_report}".format(
repo_root=Env.REPO_ROOT,
xsslint_script=xsslint_script,
xsslint_report=xsslint_report,
@@ -505,21 +506,21 @@ def run_xsslint(options):
xsslint_counts = _get_xsslint_counts(xsslint_report)
try:
metrics_str = "Number of {xsslint_script} violations: {num_violations}\n".format(
metrics_str = u"Number of {xsslint_script} violations: {num_violations}\n".format(
xsslint_script=xsslint_script, num_violations=int(xsslint_counts['total'])
)
if 'rules' in xsslint_counts and any(xsslint_counts['rules']):
metrics_str += "\n"
rule_keys = sorted(xsslint_counts['rules'].keys())
for rule in rule_keys:
metrics_str += "{rule} violations: {count}\n".format(
metrics_str += u"{rule} violations: {count}\n".format(
rule=rule,
count=int(xsslint_counts['rules'][rule])
)
except TypeError:
fail_quality(
'xsslint',
"FAILURE: Number of {xsslint_script} violations could not be found in {xsslint_report}".format(
u"FAILURE: Number of {xsslint_script} violations could not be found in {xsslint_report}".format(
xsslint_script=xsslint_script, xsslint_report=xsslint_report
)
)
@@ -528,14 +529,14 @@ def run_xsslint(options):
# Record the metric
_write_metric(metrics_str, metrics_report)
# Print number of violations to log.
sh("cat {metrics_report}".format(metrics_report=metrics_report), ignore_error=True)
sh(u"cat {metrics_report}".format(metrics_report=metrics_report), ignore_error=True)
error_message = ""
# Test total violations against threshold.
if 'total' in violation_thresholds.keys():
if violation_thresholds['total'] < xsslint_counts['total']:
error_message = "Too many violations total ({count}).\nThe limit is {violations_limit}.".format(
error_message = u"Too many violations total ({count}).\nThe limit is {violations_limit}.".format(
count=xsslint_counts['total'], violations_limit=violation_thresholds['total']
)
@@ -545,14 +546,14 @@ def run_xsslint(options):
for threshold_key in threshold_keys:
if threshold_key not in xsslint_counts['rules']:
error_message += (
"\nNumber of {xsslint_script} violations for {rule} could not be found in "
u"\nNumber of {xsslint_script} violations for {rule} could not be found in "
"{xsslint_report}."
).format(
xsslint_script=xsslint_script, rule=threshold_key, xsslint_report=xsslint_report
)
elif violation_thresholds['rules'][threshold_key] < xsslint_counts['rules'][threshold_key]:
error_message += \
"\nToo many {rule} violations ({count}).\nThe {rule} limit is {violations_limit}.".format(
u"\nToo many {rule} violations ({count}).\nThe {rule} limit is {violations_limit}.".format(
rule=threshold_key, count=xsslint_counts['rules'][threshold_key],
violations_limit=violation_thresholds['rules'][threshold_key],
)
@@ -560,7 +561,7 @@ def run_xsslint(options):
if error_message:
fail_quality(
'xsslint',
"FAILURE: XSSLinter Failed.\n{error_message}\n"
u"FAILURE: XSSLinter Failed.\n{error_message}\n"
"See {xsslint_report} or run the following command to hone in on the problem:\n"
" ./scripts/xss-commit-linter.sh -h".format(
error_message=error_message, xsslint_report=xsslint_report
@@ -583,7 +584,7 @@ def run_xsscommitlint():
_prepare_report_dir(xsscommitlint_report_dir)
sh(
"{repo_root}/scripts/{xsscommitlint_script} | tee {xsscommitlint_report}".format(
u"{repo_root}/scripts/{xsscommitlint_script} | tee {xsscommitlint_report}".format(
repo_root=Env.REPO_ROOT,
xsscommitlint_script=xsscommitlint_script,
xsscommitlint_report=xsscommitlint_report,
@@ -598,13 +599,13 @@ def run_xsscommitlint():
except TypeError:
fail_quality(
'xsscommitlint',
"FAILURE: Number of {xsscommitlint_script} violations could not be found in {xsscommitlint_report}".format(
u"FAILURE: Number of {xsscommitlint_script} violations could not be found in {xsscommitlint_report}".format(
xsscommitlint_script=xsscommitlint_script, xsscommitlint_report=xsscommitlint_report
)
)
# Print number of violations to log.
violations_count_str = "Number of {xsscommitlint_script} violations: {num_violations}\n".format(
violations_count_str = u"Number of {xsscommitlint_script} violations: {num_violations}\n".format(
xsscommitlint_script=xsscommitlint_script, num_violations=num_violations
)
@@ -612,10 +613,10 @@ def run_xsscommitlint():
metrics_report = (Env.METRICS_DIR / "xsscommitlint")
_write_metric(violations_count_str, metrics_report)
# Output report to console.
sh("cat {metrics_report}".format(metrics_report=metrics_report), ignore_error=True)
sh(u"cat {metrics_report}".format(metrics_report=metrics_report), ignore_error=True)
if num_violations:
fail_quality(
'xsscommitlint',
u'xsscommitlint',
"FAILURE: XSSCommitLinter Failed.\n{error_message}\n"
"See {xsscommitlint_report} or run the following command to hone in on the problem:\n"
" ./scripts/xss-commit-linter.sh -h".format(
@@ -669,7 +670,7 @@ def _get_report_contents(filename, report_name, last_line_only=False):
else:
return report_file.read()
else:
file_not_found_message = "FAILURE: The following log file could not be found: {file}".format(file=filename)
file_not_found_message = u"FAILURE: The following log file could not be found: {file}".format(file=filename)
fail_quality(report_name, file_not_found_message)
@@ -788,20 +789,20 @@ def run_quality(options):
if is_html:
lines = ['<body>\n']
sep = '-------------<br/>\n'
title = HTML("<h1>Quality Report: {}</h1>\n").format(linter)
title = HTML(u"<h1>Quality Report: {}</h1>\n").format(linter)
violations_bullets = ''.join(
[HTML('<li>{violation}</li><br/>\n').format(violation=violation) for violation in violations_list]
)
violations_str = HTML('<ul>\n{bullets}</ul>\n').format(bullets=HTML(violations_bullets))
violations_count_str = HTML("<b>Violations</b>: {count}<br/>\n")
fail_line = HTML("<b>FAILURE</b>: {} count should be 0<br/>\n").format(linter)
violations_count_str = HTML(u"<b>Violations</b>: {count}<br/>\n")
fail_line = HTML(u"<b>FAILURE</b>: {} count should be 0<br/>\n").format(linter)
else:
lines = []
sep = '-------------\n'
title = "Quality Report: {}\n".format(linter)
title = u"Quality Report: {}\n".format(linter)
violations_str = ''.join(violations_list)
violations_count_str = "Violations: {count}\n"
fail_line = "FAILURE: {} count should be {}\n".format(linter, limit)
violations_count_str = u"Violations: {count}\n"
fail_line = u"FAILURE: {} count should be {}\n".format(linter, limit)
violations_count_str = violations_count_str.format(count=count)
@@ -829,7 +830,7 @@ def run_quality(options):
# ----- Set up for diff-quality pylint call -----
# Set the string to be used for the diff-quality --compare-branch switch.
compare_branch = getattr(options, 'compare_branch', u'origin/master')
compare_commit = sh('git merge-base HEAD {}'.format(compare_branch), capture=True).strip()
compare_commit = sh(u'git merge-base HEAD {}'.format(compare_branch), capture=True).strip()
if sh('git rev-parse HEAD', capture=True).strip() != compare_commit:
compare_branch_string = u'--compare-branch={0}'.format(compare_commit)
@@ -881,7 +882,7 @@ def run_diff_quality(
"""
try:
sh(
"diff-quality --violations={type} "
u"diff-quality --violations={type} "
"{reports} {percentage_string} {compare_branch_string} "
"--html-report {dquality_dir}/diff_quality_{type}.html ".format(
type=violations_type,

6
pavelib/servers.py
View File

@@ -255,7 +255,7 @@ def update_db(options):
fake = "--fake-initial" if getattr(options, 'fake_initial', False) else ""
for system in ('lms', 'cms'):
# pylint: disable=line-too-long
sh("NO_EDXAPP_SUDO=1 EDX_PLATFORM_SETTINGS_OVERRIDE={settings} /edx/bin/edxapp-migrate-{system} --traceback --pythonpath=. {fake}".format(
sh(u"NO_EDXAPP_SUDO=1 EDX_PLATFORM_SETTINGS_OVERRIDE={settings} /edx/bin/edxapp-migrate-{system} --traceback --pythonpath=. {fake}".format(
settings=settings,
system=system,
fake=fake))
@@ -278,9 +278,9 @@ def check_settings(args):
settings = args.settings[0]
try:
import_cmd = "echo 'import {system}.envs.{settings}'".format(system=system, settings=settings)
import_cmd = u"echo 'import {system}.envs.{settings}'".format(system=system, settings=settings)
django_shell_cmd = django_cmd(system, settings, 'shell', '--plain', '--pythonpath=.')
sh("{import_cmd} | {shell_cmd}".format(import_cmd=import_cmd, shell_cmd=django_shell_cmd))
sh(u"{import_cmd} | {shell_cmd}".format(import_cmd=import_cmd, shell_cmd=django_shell_cmd))
except: # pylint: disable=bare-except
print("Failed to import settings", file=sys.stderr)

8
pavelib/tests.py
View File

@@ -294,7 +294,7 @@ def coverage():
if not (report_dir / '.coverage').isfile():
# This may be that the coverage files were generated using -p,
# try to combine them to the one file that we need.
sh("coverage combine --rcfile={}".format(rcfile))
sh(u"coverage combine --rcfile={}".format(rcfile))
if not os.path.getsize(report_dir / '.coverage') > 50:
# Check if the .coverage data file is larger than the base file,
@@ -309,9 +309,9 @@ def coverage():
return
# Generate the coverage.py XML report
sh("coverage xml --rcfile={}".format(rcfile))
sh(u"coverage xml --rcfile={}".format(rcfile))
# Generate the coverage.py HTML report
sh("coverage html --rcfile={}".format(rcfile))
sh(u"coverage html --rcfile={}".format(rcfile))
diff_coverage() # pylint: disable=no-value-for-parameter
@@ -347,7 +347,7 @@ def diff_coverage(options):
# Generate the diff coverage reports (HTML and console)
sh(
"diff-cover {xml_report_str} --compare-branch={compare_branch} "
u"diff-cover {xml_report_str} --compare-branch={compare_branch} "
"--html-report {diff_html_path}".format(
xml_report_str=xml_report_str,
compare_branch=compare_branch,

26
pavelib/utils/db_utils.py
View File

@@ -24,9 +24,9 @@ def remove_files_from_folder(files, folder):
file_with_path = os.path.join(folder, file_name)
try:
os.remove(file_with_path)
print('\tRemoved {}'.format(file_with_path))
print(u'\tRemoved {}'.format(file_with_path))
except OSError:
print('\tCould not remove {}. Continuing.'.format(file_with_path))
print(u'\tCould not remove {}. Continuing.'.format(file_with_path))
continue
@@ -40,9 +40,9 @@ def reset_test_db(db_cache_files, update_cache_files=True, use_existing_db=False
"""
cmd = '{}/scripts/reset-test-db.sh'.format(Env.REPO_ROOT)
if update_cache_files:
cmd = '{} --rebuild_cache'.format(cmd)
cmd = u'{} --rebuild_cache'.format(cmd)
if use_existing_db:
cmd = '{} --use-existing-db'.format(cmd)
cmd = u'{} --use-existing-db'.format(cmd)
sh(cmd)
verify_files_exist(db_cache_files)
@@ -77,7 +77,7 @@ def fingerprint_bokchoy_db_files(migration_output_files, all_db_files):
msg = "Computing the fingerprint."
print(msg)
fingerprint = compute_fingerprint(file_paths)
print("The fingerprint for bokchoy db files is: {}".format(fingerprint))
print(u"The fingerprint for bokchoy db files is: {}".format(fingerprint))
return fingerprint
@@ -100,7 +100,7 @@ def verify_files_exist(files):
for file_name in files:
file_path = os.path.join(CACHE_FOLDER, file_name)
if not os.path.isfile(file_path):
msg = "Did not find expected file: {}".format(file_path)
msg = u"Did not find expected file: {}".format(file_path)
raise BuildFailure(msg)
@@ -112,7 +112,7 @@ def calculate_bokchoy_migrations(migration_output_files):
NOTE: the script first clears out the database, then calculates
what migrations need to be run, which is all of them.
"""
sh('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT))
sh(u'{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT))
verify_files_exist(migration_output_files)
@@ -136,7 +136,7 @@ def is_fingerprint_in_bucket(fingerprint, bucket_name):
conn = boto.connect_s3(anon=True)
bucket = conn.get_bucket(bucket_name)
except Exception as e: # pylint: disable=broad-except
print("Exception caught trying to reach S3 bucket {}: {}".format(bucket_name, e))
print(u"Exception caught trying to reach S3 bucket {}: {}".format(bucket_name, e))
return False
key = boto.s3.key.Key(bucket=bucket, name=zipfile_name)
return key.exists()
@@ -158,12 +158,12 @@ def get_file_from_s3(bucket_name, zipfile_name, path):
"""
Get the file from s3 and save it to disk.
"""
print ("Retrieving {} from bucket {}.".format(zipfile_name, bucket_name))
print (u"Retrieving {} from bucket {}.".format(zipfile_name, bucket_name))
conn = boto.connect_s3(anon=True)
bucket = conn.get_bucket(bucket_name)
key = boto.s3.key.Key(bucket=bucket, name=zipfile_name)
if not key.exists():
msg = "Did not find expected file {} in the S3 bucket {}".format(
msg = u"Did not find expected file {} in the S3 bucket {}".format(
zipfile_name, bucket_name
)
raise BuildFailure(msg)
@@ -214,7 +214,7 @@ def upload_to_s3(file_name, file_path, bucket_name):
"""
Upload the specified files to an s3 bucket.
"""
print("Uploading {} to s3 bucket {}".format(file_name, bucket_name))
print(u"Uploading {} to s3 bucket {}".format(file_name, bucket_name))
try:
conn = boto.connect_s3()
except boto.exception.NoAuthHandlerFound:
@@ -230,9 +230,9 @@ def upload_to_s3(file_name, file_path, bucket_name):
key = boto.s3.key.Key(bucket=bucket, name=file_name)
bytes_written = key.set_contents_from_filename(file_path, replace=False, policy='public-read')
if bytes_written:
msg = "Wrote {} bytes to {}.".format(bytes_written, key.name)
msg = u"Wrote {} bytes to {}.".format(bytes_written, key.name)
else:
msg = "File {} already existed in bucket {}.".format(key.name, bucket_name)
msg = u"File {} already existed in bucket {}.".format(key.name, bucket_name)
print(msg)

14
pavelib/utils/envs.py
View File

@@ -35,7 +35,7 @@ def repo_root():
absolute_path = file_path.abspath()
break
except OSError:
print('Attempt {}/180 to get an absolute path failed'.format(attempt))
print(u'Attempt {}/180 to get an absolute path failed'.format(attempt))
if attempt < 180:
attempt += 1
sleep(1)
@@ -251,7 +251,7 @@ class Env(object):
django_cmd(
system,
settings,
"print_setting {django_setting} 2>{log_file}".format(
u"print_setting {django_setting} 2>{log_file}".format(
django_setting=django_setting,
log_file=cls.PRINT_SETTINGS_LOG_FILE
)
@@ -260,7 +260,7 @@ class Env(object):
)
return unicode(value).strip()
except BuildFailure:
print("Unable to print the value of the {} setting:".format(django_setting))
print(u"Unable to print the value of the {} setting:".format(django_setting))
with io.open(cls.PRINT_SETTINGS_LOG_FILE, 'r') as f:
print(f.read())
sys.exit(1)
@@ -300,8 +300,8 @@ class Env(object):
env_path = env_path.parent.parent / env_path.basename()
if not env_path.isfile():
print(
"Warning: could not find environment JSON file "
"at '{path}'".format(path=env_path),
u"Warning: could not find environment JSON file "
"at '{path}'".format(path=env_path), # pylint: disable=unicode-format-string
file=sys.stderr,
)
return dict()
@@ -313,8 +313,8 @@ class Env(object):
except ValueError:
print(
"Error: Could not parse JSON "
"in {path}".format(path=env_path),
u"Error: Could not parse JSON "
"in {path}".format(path=env_path), # pylint: disable=unicode-format-string
file=sys.stderr,
)
sys.exit(1)

10
pavelib/utils/passthrough_opts.py
View File

@@ -58,9 +58,9 @@ class PassthroughOptionParser(OptionParser):
if len(rargs) < nargs:
if nargs == 1:
self.error("%s option requires an argument" % opt)
self.error(u"%s option requires an argument" % opt)
else:
self.error("%s option requires %d arguments"
self.error(u"%s option requires %d arguments"
% (opt, nargs))
elif nargs == 1:
value = rargs.pop(0)
@@ -69,7 +69,7 @@ class PassthroughOptionParser(OptionParser):
del rargs[0:nargs]
elif had_explicit_value:
self.error("%s option does not take a value" % opt)
self.error(u"%s option does not take a value" % opt)
else:
value = None
@@ -103,9 +103,9 @@ class PassthroughOptionParser(OptionParser):
nargs = option.nargs
if len(rargs) < nargs:
if nargs == 1:
self.error("%s option requires an argument" % opt)
self.error(u"%s option requires an argument" % opt)
else:
self.error("%s option requires %d arguments"
self.error(u"%s option requires %d arguments"
% (opt, nargs))
elif nargs == 1:

2
pavelib/utils/process.py
View File

@@ -69,7 +69,7 @@ def run_multi_processes(cmd_list, out_log=None, err_log=None):
# pylint: disable=broad-except
except Exception as err:
print("Error running process {}".format(err), file=sys.stderr)
print(u"Error running process {}".format(err), file=sys.stderr)
finally:
for pid in pids:

16
pavelib/utils/test/bokchoy_utils.py
View File

@@ -42,13 +42,13 @@ def start_servers(options):
for service, info in Env.BOK_CHOY_SERVERS.iteritems():
address = "0.0.0.0:{}".format(info['port'])
cmd = ("DEFAULT_STORE={default_store} ").format(default_store=options.default_store)
cmd = (u"DEFAULT_STORE={default_store} ").format(default_store=options.default_store)
if coveragerc:
cmd += ("coverage run --rcfile={coveragerc} -m ").format(coveragerc=coveragerc)
cmd += (u"coverage run --rcfile={coveragerc} -m ").format(coveragerc=coveragerc)
else:
cmd += "python -m "
cmd += (
"manage {service} --settings {settings} runserver "
u"manage {service} --settings {settings} runserver "
"{address} --traceback --noreload".format(
service=service,
settings=Env.SETTINGS,
@@ -59,7 +59,7 @@ def start_servers(options):
for service, info in Env.BOK_CHOY_STUBS.iteritems():
cmd = (
"python -m stubs.start {service} {port} "
u"python -m stubs.start {service} {port} "
"{config}".format(
service=service,
port=info['port'],
@@ -74,7 +74,7 @@ def wait_for_server(server, port):
Wait for a server to respond with status 200
"""
print(
"Checking server {server} on port {port}".format(
u"Checking server {server} on port {port}".format(
server=server,
port=port,
)
@@ -114,7 +114,7 @@ def wait_for_test_servers():
if not ready:
msg = colorize(
"red",
"Could not contact {} test server".format(service)
u"Could not contact {} test server".format(service)
)
print(msg)
sys.exit(1)
@@ -126,7 +126,7 @@ def is_mongo_running():
"""
# The mongo command will connect to the service,
# failing with a non-zero exit code if it cannot connect.
output = os.popen('mongo --host {} --eval "print(\'running\')"'.format(Env.MONGO_HOST)).read()
output = os.popen(u'mongo --host {} --eval "print(\'running\')"'.format(Env.MONGO_HOST)).read()
return output and "running" in output
@@ -158,7 +158,7 @@ def clear_mongo():
Clears mongo database.
"""
sh(
"mongo --host {} {} --eval 'db.dropDatabase()' > /dev/null".format(
u"mongo --host {} {} --eval 'db.dropDatabase()' > /dev/null".format(
Env.MONGO_HOST,
Env.BOK_CHOY_MONGO_DATABASE,
)

16
pavelib/utils/test/suites/acceptance_suite.py
View File

@@ -49,22 +49,22 @@ def setup_acceptance_db():
# Copy the cached database to the test root directory
for db_alias in DBS:
sh("cp {db_cache} {db}".format(db_cache=DB_CACHES[db_alias], db=DBS[db_alias]))
sh(u"cp {db_cache} {db}".format(db_cache=DB_CACHES[db_alias], db=DBS[db_alias]))
# Run migrations to update the db, starting from its cached state
for db_alias in sorted(DBS):
# pylint: disable=line-too-long
sh("./manage.py lms --settings {} migrate --traceback --noinput --fake-initial --database {}".format(settings, db_alias))
sh("./manage.py cms --settings {} migrate --traceback --noinput --fake-initial --database {}".format(settings, db_alias))
sh(u"./manage.py lms --settings {} migrate --traceback --noinput --fake-initial --database {}".format(settings, db_alias))
sh(u"./manage.py cms --settings {} migrate --traceback --noinput --fake-initial --database {}".format(settings, db_alias))
else:
# If no cached database exists, migrate then create the cache
for db_alias in sorted(DBS.keys()):
sh("./manage.py lms --settings {} migrate --traceback --noinput --database {}".format(settings, db_alias))
sh("./manage.py cms --settings {} migrate --traceback --noinput --database {}".format(settings, db_alias))
sh(u"./manage.py lms --settings {} migrate --traceback --noinput --database {}".format(settings, db_alias))
sh(u"./manage.py cms --settings {} migrate --traceback --noinput --database {}".format(settings, db_alias))
# Create the cache if it doesn't already exist
for db_alias in DBS.keys():
sh("cp {db} {db_cache}".format(db_cache=DB_CACHES[db_alias], db=DBS[db_alias]))
sh(u"cp {db} {db_cache}".format(db_cache=DB_CACHES[db_alias], db=DBS[db_alias]))
class AcceptanceTest(TestSuite):
@@ -95,7 +95,7 @@ class AcceptanceTest(TestSuite):
lettuce_host = ['LETTUCE_HOST={}'.format(Env.SERVER_HOST)] if Env.USING_DOCKER else []
report_file = self.report_dir / "{}.xml".format(self.system)
report_args = ["--xunit-file {}".format(report_file)]
report_args = [u"--xunit-file {}".format(report_file)]
return lettuce_host + [
# set DBUS_SESSION_BUS_ADDRESS to avoid hangs on Chrome
"DBUS_SESSION_BUS_ADDRESS=/dev/null",
@@ -150,7 +150,7 @@ class AcceptanceTestSuite(TestSuite):
for default_store in stores:
kwargs['system'] = system
kwargs['default_store'] = default_store
self.subsuites.append(AcceptanceTest('{} acceptance using {}'.format(system, default_store), **kwargs))
self.subsuites.append(AcceptanceTest(u'{} acceptance using {}'.format(system, default_store), **kwargs))
def __enter__(self):
super(AcceptanceTestSuite, self).__enter__()

19
pavelib/utils/test/suites/bokchoy_suite.py
View File

@@ -1,3 +1,4 @@
# pylint: disable=unicode-format-string
"""
Class used for defining and running Bok Choy acceptance test suite
"""
@@ -48,7 +49,7 @@ def load_bok_choy_data(options):
"""
print('Loading data from json fixtures in db_fixtures directory')
sh(
"DEFAULT_STORE={default_store}"
u"DEFAULT_STORE={default_store}"
" ./manage.py lms --settings {settings} loaddata --traceback"
" common/test/db_fixtures/*.json".format(
default_store=options.default_store,
@@ -73,11 +74,11 @@ def load_courses(options):
`test_root/courses/`.
"""
if 'imports_dir' in options:
msg = colorize('green', "Importing courses from {}...".format(options.imports_dir))
msg = colorize('green', u"Importing courses from {}...".format(options.imports_dir))
print(msg)
sh(
"DEFAULT_STORE={default_store}"
u"DEFAULT_STORE={default_store}"
" ./manage.py cms --settings={settings} import {import_dir}".format(
default_store=options.default_store,
import_dir=options.imports_dir,
@@ -100,7 +101,7 @@ def update_fixtures():
print(msg)
sh(
" ./manage.py lms --settings={settings} update_fixtures".format(
u" ./manage.py lms --settings={settings} update_fixtures".format(
settings=Env.SETTINGS
)
)
@@ -132,7 +133,7 @@ def get_test_course(options):
print(msg)
sh(
'wget {tar_gz_file} -O {zipped_course}'.format(
u'wget {tar_gz_file} -O {zipped_course}'.format(
tar_gz_file=DEMO_COURSE_TAR_GZ,
zipped_course=zipped_course,
)
@@ -142,7 +143,7 @@ def get_test_course(options):
print(msg)
sh(
'tar zxf {zipped_course} -C {courses_dir}'.format(
u'tar zxf {zipped_course} -C {courses_dir}'.format(
zipped_course=zipped_course,
courses_dir=options.imports_dir,
)
@@ -287,7 +288,7 @@ class BokChoyTestSuite(TestSuite):
# Clean up data we created in the databases
msg = colorize('green', "Cleaning up databases...")
print(msg)
sh("./manage.py lms --settings {settings} flush --traceback --noinput".format(settings=Env.SETTINGS))
sh(u"./manage.py lms --settings {settings} flush --traceback --noinput".format(settings=Env.SETTINGS))
clear_mongo()
@property
@@ -300,7 +301,7 @@ class BokChoyTestSuite(TestSuite):
if self.num_processes != 1:
# Construct "multiprocess" pytest command
command += [
"-n {}".format(self.num_processes),
u"-n {}".format(self.num_processes),
"--color=no",
]
if self.verbosity < 1:
@@ -308,7 +309,7 @@ class BokChoyTestSuite(TestSuite):
elif self.verbosity > 1:
command.append("--verbose")
if self.eval_attr:
command.append("-a '{}'".format(self.eval_attr))
command.append(u"-a '{}'".format(self.eval_attr))
return command

19
pavelib/utils/test/suites/pytest_suite.py
View File

@@ -1,3 +1,4 @@
# pylint: disable=unicode-format-string
"""
Classes used for defining and running pytest test suites
"""
@@ -161,27 +162,27 @@ class SystemTestSuite(PytestSuite):
xdist_remote_processes = self.processes
for ip in self.xdist_ip_addresses.split(','):
# The django settings runtime command does not propagate to xdist remote workers
django_env_var_cmd = 'export DJANGO_SETTINGS_MODULE={}' \
django_env_var_cmd = u'export DJANGO_SETTINGS_MODULE={}' \
.format('{}.envs.{}'.format(self.root, self.settings))
xdist_string = '--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \
xdist_string = u'--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \
'//python="source /edx/app/edxapp/edxapp_env; {}; python"' \
'//chdir="/edx/app/edxapp/edx-platform"' \
.format(xdist_remote_processes, ip, django_env_var_cmd)
cmd.append(xdist_string)
for rsync_dir in Env.rsync_dirs():
cmd.append('--rsyncdir {}'.format(rsync_dir))
cmd.append(u'--rsyncdir {}'.format(rsync_dir))
else:
if self.processes == -1:
cmd.append('-n auto')
cmd.append('--dist=loadscope')
elif self.processes != 0:
cmd.append('-n {}'.format(self.processes))
cmd.append(u'-n {}'.format(self.processes))
cmd.append('--dist=loadscope')
if not self.randomize:
cmd.append('-p no:randomly')
if self.eval_attr:
cmd.append("-a '{}'".format(self.eval_attr))
cmd.append(u"-a '{}'".format(self.eval_attr))
cmd.extend(self.passthrough_options)
cmd.append(self.test_id)
@@ -284,25 +285,25 @@ class LibTestSuite(PytestSuite):
django_env_var_cmd = "export DJANGO_SETTINGS_MODULE='lms.envs.test'"
else:
django_env_var_cmd = "export DJANGO_SETTINGS_MODULE='openedx.tests.settings'"
xdist_string = '--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \
xdist_string = u'--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \
'//python="source /edx/app/edxapp/edxapp_env; {}; python"' \
'//chdir="/edx/app/edxapp/edx-platform"' \
.format(xdist_remote_processes, ip, django_env_var_cmd)
cmd.append(xdist_string)
for rsync_dir in Env.rsync_dirs():
cmd.append('--rsyncdir {}'.format(rsync_dir))
cmd.append(u'--rsyncdir {}'.format(rsync_dir))
else:
if self.processes == -1:
cmd.append('-n auto')
cmd.append('--dist=loadscope')
elif self.processes != 0:
cmd.append('-n {}'.format(self.processes))
cmd.append(u'-n {}'.format(self.processes))
cmd.append('--dist=loadscope')
if not self.randomize:
cmd.append("-p no:randomly")
if self.eval_attr:
cmd.append("-a '{}'".format(self.eval_attr))
cmd.append(u"-a '{}'".format(self.eval_attr))
cmd.append(self.test_id)

9
pavelib/utils/test/suites/suite.py
View File

@@ -1,3 +1,4 @@
# pylint: disable=unicode-format-string
"""
A class used for defining and running test suites
"""
@@ -39,7 +40,7 @@ class TestSuite(object):
i.e. Checking for and defining required directories.
"""
print("\nSetting up for {suite_name}".format(suite_name=self.root))
print(u"\nSetting up for {suite_name}".format(suite_name=self.root))
self.failed_suites = []
def __exit__(self, exc_type, exc_value, traceback):
@@ -52,7 +53,7 @@ class TestSuite(object):
i.e. Cleaning mongo after the lms tests run.
"""
print("\nCleaning up after {suite_name}".format(suite_name=self.root))
print(u"\nCleaning up after {suite_name}".format(suite_name=self.root))
@property
def cmd(self):
@@ -123,10 +124,10 @@ class TestSuite(object):
Writes a list of failed_suites to sys.stderr
"""
if self.failed_suites:
msg = colorize('red', "\n\n{bar}\nTests failed in the following suites:\n* ".format(bar="=" * 48))
msg = colorize('red', u"\n\n{bar}\nTests failed in the following suites:\n* ".format(bar="=" * 48))
msg += colorize('red', '\n* '.join([s.root for s in self.failed_suites]) + '\n\n')
else:
msg = colorize('green', "\n\n{bar}\nNo test failures ".format(bar="=" * 48))
msg = colorize('green', u"\n\n{bar}\nNo test failures ".format(bar="=" * 48))
print(msg)

32
pavelib/utils/test/utils.py
View File

@@ -32,7 +32,7 @@ def clean_test_files():
# This find command removes all the *.pyc files that aren't in the .git
# directory. See this blog post for more details:
# http://nedbatchelder.com/blog/201505/be_careful_deleting_files_around_git.html
sh(r"find . -name '.git' -prune -o -name '*.pyc' -exec rm {} \;")
sh(r"find . -name '.git' -prune -o -name '*.pyc' -exec rm {} \;") # pylint: disable=unicode-format-string
sh("rm -rf test_root/log/auto_screenshots/*")
sh("rm -rf /tmp/mako_[cl]ms")
@@ -55,7 +55,7 @@ def clean_dir(directory):
"""
# We delete the files but preserve the directory structure
# so that coverage.py has a place to put the reports.
sh('find {dir} -type f -delete'.format(dir=directory))
sh(u'find {dir} -type f -delete'.format(dir=directory))
@task
@@ -84,7 +84,7 @@ def clean_mongo():
"""
Clean mongo test databases
"""
sh("mongo {host}:{port} {repo_root}/scripts/delete-mongo-test-dbs.js".format(
sh(u"mongo {host}:{port} {repo_root}/scripts/delete-mongo-test-dbs.js".format(
host=Env.MONGO_HOST,
port=MONGO_PORT_NUM,
repo_root=Env.REPO_ROOT,
@@ -111,10 +111,10 @@ def check_firefox_version():
driver.close()
if firefox_ver < MINIMUM_FIREFOX_VERSION:
raise Exception(
'Required firefox version not found.\n'
'Expected: {expected_version}; Actual: {actual_version}.\n\n'
'Make sure that the edx.devstack.firefox container is up-to-date and running\n'
'\t{expected_version}'.format(
u'Required firefox version not found.\n'
u'Expected: {expected_version}; Actual: {actual_version}.\n\n'
u'Make sure that the edx.devstack.firefox container is up-to-date and running\n'
u'\t{expected_version}'.format(
actual_version=firefox_ver,
expected_version=MINIMUM_FIREFOX_VERSION
)
@@ -135,15 +135,15 @@ def check_firefox_version():
if firefox_ver < MINIMUM_FIREFOX_VERSION:
raise Exception(
'Required firefox version not found.\n'
'Expected: {expected_version}; Actual: {actual_version}.\n\n'
'As the vagrant user in devstack, run the following:\n\n'
'\t$ sudo wget -O /tmp/firefox_42.deb {debian_path}\n'
'\t$ sudo apt-get remove firefox\n\n'
'\t$ sudo gdebi -nq /tmp/firefox_42.deb\n\n'
'Confirm the new version:\n'
'\t$ firefox --version\n'
'\t{expected_version}'.format(
u'Required firefox version not found.\n'
u'Expected: {expected_version}; Actual: {actual_version}.\n\n'
u'As the vagrant user in devstack, run the following:\n\n'
u'\t$ sudo wget -O /tmp/firefox_42.deb {debian_path}\n'
u'\t$ sudo apt-get remove firefox\n\n'
u'\t$ sudo gdebi -nq /tmp/firefox_42.deb\n\n'
u'Confirm the new version:\n'
u'\t$ firefox --version\n'
u'\t{expected_version}'.format(
actual_version=firefox_ver,
expected_version=expected_firefox_ver,
debian_path=debian_path