From 6cf44a04670f2380a04acfd67b024a8b087035a2 Mon Sep 17 00:00:00 2001 From: Matthew Piatetsky Date: Fri, 1 Feb 2019 16:44:26 -0500 Subject: [PATCH] fix unicode strings in pavelib/ --- pavelib/acceptance_test.py | 2 +- pavelib/assets.py | 46 +++++------ pavelib/bok_choy.py | 10 +-- pavelib/database.py | 4 +- pavelib/docs.py | 10 +-- pavelib/i18n.py | 6 +- pavelib/js_test.py | 2 +- pavelib/paver_tests/test_assets.py | 6 +- pavelib/paver_tests/test_database.py | 12 +-- .../paver_tests/test_extract_and_generate.py | 6 +- .../paver_tests/test_paver_bok_choy_cmds.py | 8 +- pavelib/paver_tests/test_paver_pytest_cmds.py | 11 +-- pavelib/paver_tests/test_prereqs.py | 2 +- pavelib/paver_tests/test_servers.py | 8 +- pavelib/paver_tests/test_utils.py | 4 +- pavelib/prereqs.py | 10 +-- pavelib/quality.py | 79 ++++++++++--------- pavelib/servers.py | 6 +- pavelib/tests.py | 8 +- pavelib/utils/db_utils.py | 26 +++--- pavelib/utils/envs.py | 14 ++-- pavelib/utils/passthrough_opts.py | 10 +-- pavelib/utils/process.py | 2 +- pavelib/utils/test/bokchoy_utils.py | 16 ++-- pavelib/utils/test/suites/acceptance_suite.py | 16 ++-- pavelib/utils/test/suites/bokchoy_suite.py | 19 ++--- pavelib/utils/test/suites/pytest_suite.py | 19 ++--- pavelib/utils/test/suites/suite.py | 9 ++- pavelib/utils/test/utils.py | 32 ++++---- 29 files changed, 204 insertions(+), 199 deletions(-) diff --git a/pavelib/acceptance_test.py b/pavelib/acceptance_test.py index 68555def6f..afa24c56c9 100644 --- a/pavelib/acceptance_test.py +++ b/pavelib/acceptance_test.py @@ -61,5 +61,5 @@ def test_acceptance(options, passthrough_options): ) print(msg) - suite = AcceptanceTestSuite('{} acceptance'.format(opts['system']), **opts) + suite = AcceptanceTestSuite(u'{} acceptance'.format(opts['system']), **opts) suite.run() diff --git a/pavelib/assets.py b/pavelib/assets.py index 490966e60b..23e8bbd641 100644 --- a/pavelib/assets.py +++ b/pavelib/assets.py @@ -89,7 +89,7 @@ SASS_LOOKUP_DEPENDENCIES = { COLLECTSTATIC_LOG_DIR_ARG = 'collect_log_dir' # Webpack command -WEBPACK_COMMAND = 'STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} $(npm bin)/webpack {options}' +WEBPACK_COMMAND = u'STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} $(npm bin)/webpack {options}' def get_sass_directories(system, theme_dir=None): @@ -110,7 +110,7 @@ def get_sass_directories(system, theme_dir=None): :param theme_dir: absolute path of theme for which to compile sass files. """ if system not in SYSTEMS: - raise ValueError("'system' must be one of ({allowed_values})".format(allowed_values=', '.join(SYSTEMS.keys()))) + raise ValueError(u"'system' must be one of ({allowed_values})".format(allowed_values=', '.join(SYSTEMS.keys()))) system = SYSTEMS[system] applicable_directories = list() @@ -453,7 +453,7 @@ def compile_sass(options): for system in systems: for theme in themes: - print("Started compiling '{system}' Sass for '{theme}'.".format(system=system, theme=theme or 'system')) + print(u"Started compiling '{system}' Sass for '{theme}'.".format(system=system, theme=theme or 'system')) # Compile sass files is_successful = _compile_sass( @@ -465,18 +465,18 @@ def compile_sass(options): ) if is_successful: - print("Finished compiling '{system}' Sass for '{theme}'.".format( + print(u"Finished compiling '{system}' Sass for '{theme}'.".format( system=system, theme=theme or 'system' )) - compilation_results['success' if is_successful else 'failure'].append('{system} sass for {theme}.'.format( + compilation_results['success' if is_successful else 'failure'].append(u'{system} sass for {theme}.'.format( system=system, theme=theme or 'system', )) print("\t\tFinished compiling Sass:") if not dry_run: for sass_dir, css_dir, duration in timing_info: - print(">> {} -> {} in {}s".format(sass_dir, css_dir, duration)) + print(u">> {} -> {} in {}s".format(sass_dir, css_dir, duration)) if compilation_results['success']: print("\033[92m\nSuccessful compilations:\n--- " + "\n--- ".join(compilation_results['success']) + "\n\033[00m") @@ -522,7 +522,7 @@ def _compile_sass(system, theme, debug, force, timing_info): lookup_paths = dirs['lookup_paths'] if not sass_source_dir.isdir(): - print("\033[91m Sass dir '{dir}' does not exists, skipping sass compilation for '{theme}' \033[00m".format( + print(u"\033[91m Sass dir '{dir}' does not exists, skipping sass compilation for '{theme}' \033[00m".format( dir=sass_dirs, theme=theme or system, )) # theme doesn't override sass directory, so skip it @@ -530,14 +530,14 @@ def _compile_sass(system, theme, debug, force, timing_info): if force: if dry_run: - tasks.environment.info("rm -rf {css_dir}/*.css".format( + tasks.environment.info(u"rm -rf {css_dir}/*.css".format( css_dir=css_dir, )) else: - sh("rm -rf {css_dir}/*.css".format(css_dir=css_dir)) + sh(u"rm -rf {css_dir}/*.css".format(css_dir=css_dir)) if dry_run: - tasks.environment.info("libsass {sass_dir}".format( + tasks.environment.info(u"libsass {sass_dir}".format( sass_dir=sass_source_dir, )) else: @@ -554,7 +554,7 @@ def _compile_sass(system, theme, debug, force, timing_info): if should_generate_rtl_css_file(sass_file): source_css_file = sass_file.replace(sass_source_dir, css_dir).replace('.scss', '.css') target_css_file = source_css_file.replace('.css', '-rtl.css') - sh("rtlcss {source_file} {target_file}".format( + sh(u"rtlcss {source_file} {target_file}".format( source_file=source_css_file, target_file=target_css_file, )) @@ -604,19 +604,19 @@ def process_npm_assets(): else: vendor_dir = NPM_JS_VENDOR_DIRECTORY if os.path.exists(library_path): - sh('/bin/cp -rf {library_path} {vendor_dir}'.format( + sh(u'/bin/cp -rf {library_path} {vendor_dir}'.format( library_path=library_path, vendor_dir=vendor_dir, )) elif not skip_if_missing: - raise Exception('Missing vendor file {library_path}'.format(library_path=library_path)) + raise Exception(u'Missing vendor file {library_path}'.format(library_path=library_path)) def copy_vendor_library_dir(library_dir, skip_if_missing=False): """ Copies all vendor libraries in directory to the shared vendor directory. """ library_dir_path = 'node_modules/{library_dir}'.format(library_dir=library_dir) - print('Copying vendor library dir: {}'.format(library_dir_path)) + print(u'Copying vendor library dir: {}'.format(library_dir_path)) if os.path.exists(library_dir_path): for dirpath, _, filenames in os.walk(library_dir_path): for filename in filenames: @@ -695,16 +695,16 @@ def collect_assets(systems, settings, **kwargs): ] ignore_args = " ".join( - '--ignore "{}"'.format(pattern) for pattern in ignore_patterns + u'--ignore "{}"'.format(pattern) for pattern in ignore_patterns ) for sys in systems: collectstatic_stdout_str = _collect_assets_cmd(sys, **kwargs) - sh(django_cmd(sys, settings, "collectstatic {ignore_args} --noinput {logfile_str}".format( + sh(django_cmd(sys, settings, u"collectstatic {ignore_args} --noinput {logfile_str}".format( ignore_args=ignore_args, logfile_str=collectstatic_stdout_str ))) - print("\t\tFinished collecting {} assets.".format(sys)) + print(u"\t\tFinished collecting {} assets.".format(sys)) def _collect_assets_cmd(system, **kwargs): @@ -717,7 +717,7 @@ def _collect_assets_cmd(system, **kwargs): if kwargs[COLLECTSTATIC_LOG_DIR_ARG] is None: collectstatic_stdout_str = "" else: - collectstatic_stdout_str = "> {output_dir}/{sys}-collectstatic.log".format( + collectstatic_stdout_str = u"> {output_dir}/{sys}-collectstatic.log".format( output_dir=kwargs[COLLECTSTATIC_LOG_DIR_ARG], sys=system ) @@ -743,7 +743,7 @@ def execute_compile_sass(args): django_cmd( sys, args.settings, - "compile_sass {system} {options}".format( + u"compile_sass {system} {options}".format( system='cms' if sys == 'studio' else sys, options=options, ), @@ -765,14 +765,14 @@ def webpack(options): static_root_lms = Env.get_django_setting("STATIC_ROOT", "lms", settings=settings) static_root_cms = Env.get_django_setting("STATIC_ROOT", "cms", settings=settings) config_path = Env.get_django_setting("WEBPACK_CONFIG_PATH", "lms", settings=settings) - environment = 'NODE_ENV={node_env} STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms}'.format( + environment = u'NODE_ENV={node_env} STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms}'.format( node_env="development" if config_path == 'webpack.dev.config.js' else "production", static_root_lms=static_root_lms, static_root_cms=static_root_cms ) sh( cmd( - '{environment} $(npm bin)/webpack --config={config_path}'.format( + u'{environment} $(npm bin)/webpack --config={config_path}'.format( environment=environment, config_path=config_path ) @@ -788,8 +788,8 @@ def execute_webpack_watch(settings=None): # not all JS files, so we use its own watcher instead of subclassing # from Watchdog like the other watchers do. run_background_process( - 'STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} $(npm bin)/webpack {options}'.format( - options='--watch --config={config_path}'.format( + u'STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} $(npm bin)/webpack {options}'.format( + options=u'--watch --config={config_path}'.format( config_path=Env.get_django_setting("WEBPACK_CONFIG_PATH", "lms", settings=settings) ), static_root_lms=Env.get_django_setting("STATIC_ROOT", "lms", settings=settings), diff --git a/pavelib/bok_choy.py b/pavelib/bok_choy.py index 7ddab963fe..9d21b068bd 100644 --- a/pavelib/bok_choy.py +++ b/pavelib/bok_choy.py @@ -143,7 +143,7 @@ def run_bokchoy(options, passthrough_options): test_suite = BokChoyTestSuite('bok-choy', passthrough_options=passthrough_options, **options) msg = colorize( 'green', - 'Running tests using {default_store} modulestore.'.format( + u'Running tests using {default_store} modulestore.'.format( default_store=test_suite.default_store, ) ) @@ -160,14 +160,14 @@ def parse_coverage(report_dir, coveragerc): msg = colorize('green', "Combining coverage reports") print(msg) - sh("coverage combine --rcfile={}".format(coveragerc)) + sh(u"coverage combine --rcfile={}".format(coveragerc)) msg = colorize('green', "Generating coverage reports") print(msg) - sh("coverage html --rcfile={}".format(coveragerc)) - sh("coverage xml --rcfile={}".format(coveragerc)) - sh("coverage report --rcfile={}".format(coveragerc)) + sh(u"coverage html --rcfile={}".format(coveragerc)) + sh(u"coverage xml --rcfile={}".format(coveragerc)) + sh(u"coverage report --rcfile={}".format(coveragerc)) @task diff --git a/pavelib/database.py b/pavelib/database.py index 6ab05b95d7..2064a199fa 100644 --- a/pavelib/database.py +++ b/pavelib/database.py @@ -103,7 +103,7 @@ def update_local_bokchoy_db_from_s3(options): reset_test_db(BOKCHOY_DB_FILES, update_cache_files=False, use_existing_db=True) else: - msg = "{} {} {}".format( + msg = u"{} {} {}".format( "Did not find updated bokchoy db files at S3.", "Loading the bokchoy db files from disk", "and running migrations." @@ -116,7 +116,7 @@ def update_local_bokchoy_db_from_s3(options): if not is_fingerprint_in_bucket(fingerprint, CACHE_BUCKET_NAME): upload_db_cache_to_s3(fingerprint, BOKCHOY_DB_FILES, CACHE_BUCKET_NAME) else: - msg = "{} {}. {}".format( + msg = u"{} {}. {}".format( "Found a matching fingerprint in bucket ", CACHE_BUCKET_NAME, "Not pushing to s3" diff --git a/pavelib/docs.py b/pavelib/docs.py index 54343f236b..a7a55af382 100644 --- a/pavelib/docs.py +++ b/pavelib/docs.py @@ -40,8 +40,8 @@ def doc_path(options, allow_default=True): if doc_type == 'default' and not allow_default: print( - "You must specify a documentation type using '--type'. " - "Valid options are: {options}".format( + u"You must specify a documentation type using '--type'. " + u"Valid options are: {options}".format( options=valid_doc_types() ) ) @@ -49,8 +49,8 @@ def doc_path(options, allow_default=True): if path is None: print( - "Invalid documentation type '{doc_type}'. " - "Valid options are: {options}".format( + u"Invalid documentation type '{doc_type}'. " + u"Valid options are: {options}".format( doc_type=doc_type, options=valid_doc_types() ) ) @@ -73,7 +73,7 @@ def build_docs(options): """ verbose = getattr(options, 'verbose', False) - cmd = "cd {dir}; make html quiet={quiet}".format( + cmd = u"cd {dir}; make html quiet={quiet}".format( dir=doc_path(options), quiet="false" if verbose else "true" ) diff --git a/pavelib/i18n.py b/pavelib/i18n.py index 5eb095ce29..951d65735e 100644 --- a/pavelib/i18n.py +++ b/pavelib/i18n.py @@ -131,9 +131,9 @@ def i18n_validate_transifex_config(): if not config.isfile or config.getsize == 0: msg = colorize( 'red', - "Cannot connect to Transifex, config file is missing" - " or empty: {config} \nSee " - "http://help.transifex.com/features/client/#transifexrc \n".format( + u"Cannot connect to Transifex, config file is missing" + u" or empty: {config} \nSee " + u"http://help.transifex.com/features/client/#transifexrc \n".format( config=config, ) ) diff --git a/pavelib/js_test.py b/pavelib/js_test.py index a04d32eadc..5b95013225 100644 --- a/pavelib/js_test.py +++ b/pavelib/js_test.py @@ -48,7 +48,7 @@ def test_js(options): if (suite != 'all') and (suite not in Env.JS_TEST_ID_KEYS): sys.stderr.write( - "Unknown test suite. Please choose from ({suites})\n".format( + u"Unknown test suite. Please choose from ({suites})\n".format( suites=", ".join(Env.JS_TEST_ID_KEYS) ) ) diff --git a/pavelib/paver_tests/test_assets.py b/pavelib/paver_tests/test_assets.py index 91463cf62d..2df98064b6 100644 --- a/pavelib/paver_tests/test_assets.py +++ b/pavelib/paver_tests/test_assets.py @@ -147,7 +147,7 @@ class TestPaverThemeAssetTasks(PaverTestCase): ) if force: expected_messages.append( - 'rm -rf {test_theme_dir}/lms/static/css/*.css'.format(test_theme_dir=str(TEST_THEME_DIR)) + u'rm -rf {test_theme_dir}/lms/static/css/*.css'.format(test_theme_dir=str(TEST_THEME_DIR)) ) expected_messages.append( u'libsass {test_theme_dir}/lms/static/sass'.format(test_theme_dir=str(TEST_THEME_DIR)) @@ -345,7 +345,7 @@ class TestCollectAssets(PaverTestCase): """ for i, sys in enumerate(systems): msg = self.task_messages[i] - self.assertTrue(msg.startswith('python manage.py {}'.format(sys))) + self.assertTrue(msg.startswith(u'python manage.py {}'.format(sys))) self.assertIn(' collectstatic ', msg) self.assertIn('--settings={}'.format(Env.DEVSTACK_SETTINGS), msg) self.assertTrue(msg.endswith(' {}'.format(log_location))) @@ -372,7 +372,7 @@ class TestUpdateAssetsTask(PaverTestCase): call_task('pavelib.assets.update_assets', args=cmd_args) self.assertTrue( self._is_substring_in_list(self.task_messages, expected_substring), - msg="{substring} not found in messages".format(substring=expected_substring) + msg=u"{substring} not found in messages".format(substring=expected_substring) ) def _is_substring_in_list(self, messages_list, expected_substring): diff --git a/pavelib/paver_tests/test_database.py b/pavelib/paver_tests/test_database.py index 4ff77d0deb..f3e5c31f77 100644 --- a/pavelib/paver_tests/test_database.py +++ b/pavelib/paver_tests/test_database.py @@ -115,8 +115,8 @@ class TestPaverDatabaseTasks(MockS3Mixin, PaverTestCase): # Make sure that the local cache files are used - NOT downloaded from s3 self.assertFalse(_mock_get_file.called) calls = [ - call('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)), - call('{}/scripts/reset-test-db.sh --use-existing-db'.format(Env.REPO_ROOT)) + call(u'{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)), + call(u'{}/scripts/reset-test-db.sh --use-existing-db'.format(Env.REPO_ROOT)) ] _mock_sh.assert_has_calls(calls) @@ -156,8 +156,8 @@ class TestPaverDatabaseTasks(MockS3Mixin, PaverTestCase): 'moto_test_bucket', self.fingerprint_filename, db_utils.CACHE_FOLDER ) calls = [ - call('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)), - call('{}/scripts/reset-test-db.sh --use-existing-db'.format(Env.REPO_ROOT)) + call(u'{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)), + call(u'{}/scripts/reset-test-db.sh --use-existing-db'.format(Env.REPO_ROOT)) ] _mock_sh.assert_has_calls(calls) @@ -184,8 +184,8 @@ class TestPaverDatabaseTasks(MockS3Mixin, PaverTestCase): database.update_local_bokchoy_db_from_s3() # pylint: disable=no-value-for-parameter calls = [ - call('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)), - call('{}/scripts/reset-test-db.sh --rebuild_cache --use-existing-db'.format(Env.REPO_ROOT)) + call(u'{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)), + call(u'{}/scripts/reset-test-db.sh --rebuild_cache --use-existing-db'.format(Env.REPO_ROOT)) ] _mock_sh.assert_has_calls(calls) diff --git a/pavelib/paver_tests/test_extract_and_generate.py b/pavelib/paver_tests/test_extract_and_generate.py index 9646b84cbf..2ab64036aa 100644 --- a/pavelib/paver_tests/test_extract_and_generate.py +++ b/pavelib/paver_tests/test_extract_and_generate.py @@ -84,11 +84,11 @@ class TestGenerate(TestCase): mofile = filename + '.mo' path = os.path.join(self.configuration.get_messages_dir(locale), mofile) exists = os.path.exists(path) - self.assertTrue(exists, msg='Missing file in locale %s: %s' % (locale, mofile)) + self.assertTrue(exists, msg=u'Missing file in locale %s: %s' % (locale, mofile)) self.assertGreaterEqual( datetime.fromtimestamp(os.path.getmtime(path), UTC), self.start_time, - msg='File not recently modified: %s' % path + msg=u'File not recently modified: %s' % path ) # Segmenting means that the merge headers don't work they way they # used to, so don't make this check for now. I'm not sure if we'll @@ -113,7 +113,7 @@ class TestGenerate(TestCase): self.assertEqual( len(match), 3, - msg="Found %s (should be 3) merge comments in the header for %s" % (len(match), path) + msg=u"Found %s (should be 3) merge comments in the header for %s" % (len(match), path) ) diff --git a/pavelib/paver_tests/test_paver_bok_choy_cmds.py b/pavelib/paver_tests/test_paver_bok_choy_cmds.py index 6de264b719..7865240ddd 100644 --- a/pavelib/paver_tests/test_paver_bok_choy_cmds.py +++ b/pavelib/paver_tests/test_paver_bok_choy_cmds.py @@ -145,7 +145,7 @@ class TestPaverBokChoyCmd(unittest.TestCase): repo_dir=REPO_DIR, shard_str='/shard_' + self.shard if self.shard else '', ), - "-n {}".format(process_count), + u"-n {}".format(process_count), "--color=no", "--verbose", ] @@ -162,7 +162,7 @@ class TestPaverBokChoyCmd(unittest.TestCase): repo_dir=REPO_DIR, shard_str='/shard_' + self.shard if self.shard else '', ), - "-n {}".format(process_count), + u"-n {}".format(process_count), "--color=no", "--verbose", ] @@ -244,9 +244,9 @@ class TestPaverPa11yCrawlerCmd(unittest.TestCase): else: self._mock_sh.assert_has_calls([ call( - 'wget {targz} -O {dir}demo_course.tar.gz'.format(targz=DEMO_COURSE_TAR_GZ, dir=downloaded_to)), + u'wget {targz} -O {dir}demo_course.tar.gz'.format(targz=DEMO_COURSE_TAR_GZ, dir=downloaded_to)), call( - 'tar zxf {dir}demo_course.tar.gz -C {dir}'.format(dir=downloaded_to)), + u'tar zxf {dir}demo_course.tar.gz -C {dir}'.format(dir=downloaded_to)), ]) @patch("pavelib.utils.test.suites.bokchoy_suite.path") diff --git a/pavelib/paver_tests/test_paver_pytest_cmds.py b/pavelib/paver_tests/test_paver_pytest_cmds.py index 9a25f15c5a..b2bb55a3f5 100644 --- a/pavelib/paver_tests/test_paver_pytest_cmds.py +++ b/pavelib/paver_tests/test_paver_pytest_cmds.py @@ -1,3 +1,4 @@ +# pylint: disable=unicode-format-string """ Tests for the pytest paver commands themselves. Run just this test with: paver test_lib -t pavelib/paver_tests/test_paver_pytest_cmds.py @@ -47,26 +48,26 @@ class TestPaverPytestCmd(unittest.TestCase): processes = 1 if pytestSubclass == "SystemTestSuite": - django_env_var_cmd = "export DJANGO_SETTINGS_MODULE={}.envs.test".format(root) + django_env_var_cmd = u"export DJANGO_SETTINGS_MODULE={}.envs.test".format(root) elif pytestSubclass == "LibTestSuite": if 'pavelib/paver_tests' in test_id: - django_env_var_cmd = "export DJANGO_SETTINGS_MODULE={}.envs.test".format(root) + django_env_var_cmd = u"export DJANGO_SETTINGS_MODULE={}.envs.test".format(root) else: django_env_var_cmd = "export DJANGO_SETTINGS_MODULE='openedx.tests.settings'" - xdist_string = '--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \ + xdist_string = u'--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \ '//python="source /edx/app/edxapp/edxapp_env; {}; python"' \ '//chdir="/edx/app/edxapp/edx-platform"' \ .format(processes, ip, django_env_var_cmd) expected_statement.append(xdist_string) for rsync_dir in Env.rsync_dirs(): - expected_statement.append('--rsyncdir {}'.format(rsync_dir)) + expected_statement.append(u'--rsyncdir {}'.format(rsync_dir)) else: if processes == -1: expected_statement.append('-n auto') expected_statement.append('--dist=loadscope') elif processes != 0: - expected_statement.append('-n {}'.format(processes)) + expected_statement.append(u'-n {}'.format(processes)) expected_statement.append('--dist=loadscope') expected_statement.extend([ diff --git a/pavelib/paver_tests/test_prereqs.py b/pavelib/paver_tests/test_prereqs.py index 41a299c870..c1c3c3e0ce 100644 --- a/pavelib/paver_tests/test_prereqs.py +++ b/pavelib/paver_tests/test_prereqs.py @@ -30,7 +30,7 @@ class TestPaverPrereqInstall(unittest.TestCase): self.assertEqual( pavelib.prereqs.no_prereq_install(), expected_val, - 'NO_PREREQ_INSTALL is set to {}, but we read it as {}'.format( + u'NO_PREREQ_INSTALL is set to {}, but we read it as {}'.format( set_val, expected_val), ) diff --git a/pavelib/paver_tests/test_servers.py b/pavelib/paver_tests/test_servers.py index 3016febb36..e562ecd26b 100644 --- a/pavelib/paver_tests/test_servers.py +++ b/pavelib/paver_tests/test_servers.py @@ -48,7 +48,7 @@ EXPECTED_PRINT_SETTINGS_COMMAND = [ ] EXPECTED_WEBPACK_COMMAND = ( u"NODE_ENV={node_env} STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} " - "$(npm bin)/webpack --config={webpack_config_path}" + u"$(npm bin)/webpack --config={webpack_config_path}" ) @@ -167,7 +167,7 @@ class TestPaverServerTasks(PaverTestCase): settings = options.get("settings", Env.DEVSTACK_SETTINGS) call_task("pavelib.servers.update_db", options=options) # pylint: disable=line-too-long - db_command = "NO_EDXAPP_SUDO=1 EDX_PLATFORM_SETTINGS_OVERRIDE={settings} /edx/bin/edxapp-migrate-{server} --traceback --pythonpath=. " + db_command = u"NO_EDXAPP_SUDO=1 EDX_PLATFORM_SETTINGS_OVERRIDE={settings} /edx/bin/edxapp-migrate-{server} --traceback --pythonpath=. " self.assertEquals( self.task_messages, [ @@ -192,8 +192,8 @@ class TestPaverServerTasks(PaverTestCase): self.assertEquals( self.task_messages, [ - "echo 'import {system}.envs.{settings}' " - "| python manage.py {system} --settings={settings} shell --plain --pythonpath=.".format( + u"echo 'import {system}.envs.{settings}' " + u"| python manage.py {system} --settings={settings} shell --plain --pythonpath=.".format( system=system, settings=settings ), ] diff --git a/pavelib/paver_tests/test_utils.py b/pavelib/paver_tests/test_utils.py index cccd922f74..28b331ee89 100644 --- a/pavelib/paver_tests/test_utils.py +++ b/pavelib/paver_tests/test_utils.py @@ -19,7 +19,7 @@ class TestUtils(unittest.TestCase): @patch('subprocess.check_output') def test_firefox_version_ok(self, _mock_subprocesss): test_version = MINIMUM_FIREFOX_VERSION - _mock_subprocesss.return_value = "Mozilla Firefox {version}".format( + _mock_subprocesss.return_value = u"Mozilla Firefox {version}".format( version=str(test_version) ) # No exception should be raised @@ -28,7 +28,7 @@ class TestUtils(unittest.TestCase): @patch('subprocess.check_output') def test_firefox_version_below_expected(self, _mock_subprocesss): test_version = MINIMUM_FIREFOX_VERSION - 1 - _mock_subprocesss.return_value = "Mozilla Firefox {version}".format( + _mock_subprocesss.return_value = u"Mozilla Firefox {version}".format( version=test_version ) with self.assertRaises(Exception): diff --git a/pavelib/prereqs.py b/pavelib/prereqs.py index b674a3774a..1f58672c1d 100644 --- a/pavelib/prereqs.py +++ b/pavelib/prereqs.py @@ -121,7 +121,7 @@ def prereq_cache(cache_name, paths, install_func): post_install_hash = compute_fingerprint(paths) cache_file.write(post_install_hash) else: - print('{cache} unchanged, skipping...'.format(cache=cache_name)) + print(u'{cache} unchanged, skipping...'.format(cache=cache_name)) def node_prereqs_installation(): @@ -157,7 +157,7 @@ def node_prereqs_installation(): proc.wait() else: raise BuildFailure(error_text) - print("Successfully installed NPM packages. Log found at {}".format( + print(u"Successfully installed NPM packages. Log found at {}".format( npm_log_file_path )) @@ -173,7 +173,7 @@ def python_prereqs_installation(): def pip_install_req_file(req_file): """Pip install the requirements file.""" pip_cmd = 'pip install -q --disable-pip-version-check --exists-action w' - sh("{pip_cmd} -r {req_file}".format(pip_cmd=pip_cmd, req_file=req_file)) + sh(u"{pip_cmd} -r {req_file}".format(pip_cmd=pip_cmd, req_file=req_file)) @task @@ -243,7 +243,7 @@ def uninstall_python_packages(): for package_name in PACKAGES_TO_UNINSTALL: if package_in_frozen(package_name, frozen): # Uninstall the pacakge - sh("pip uninstall --disable-pip-version-check -y {}".format(package_name)) + sh(u"pip uninstall --disable-pip-version-check -y {}".format(package_name)) uninstalled = True if not uninstalled: break @@ -337,7 +337,7 @@ def install_prereqs(): def log_installed_python_prereqs(): """ Logs output of pip freeze for debugging. """ - sh("pip freeze > {}".format(Env.GEN_LOG_DIR + "/pip_freeze.log")) + sh(u"pip freeze > {}".format(Env.GEN_LOG_DIR + "/pip_freeze.log")) return diff --git a/pavelib/quality.py b/pavelib/quality.py index bf43558d49..3ab6531bce 100644 --- a/pavelib/quality.py +++ b/pavelib/quality.py @@ -1,3 +1,4 @@ +# pylint: disable=unicode-format-string # coding=utf-8 """ @@ -18,12 +19,12 @@ from .utils.envs import Env from .utils.timer import timed ALL_SYSTEMS = 'lms,cms,common,openedx,pavelib' -JUNIT_XML_TEMPLATE = """ +JUNIT_XML_TEMPLATE = u""" {failure_element} """ -JUNIT_XML_FAILURE_TEMPLATE = '' +JUNIT_XML_FAILURE_TEMPLATE = u'' START_TIME = datetime.utcnow() @@ -97,7 +98,7 @@ def find_fixme(options): apps_list = ' '.join(top_python_dirs(system)) cmd = ( - "pylint --disable all --enable=fixme " + u"pylint --disable all --enable=fixme " "--output-format=parseable {apps} " "> {report_dir}/pylint_fixme.report".format( apps=apps_list, @@ -139,7 +140,7 @@ def _get_pylint_violations(systems=ALL_SYSTEMS.split(','), errors_only=False, cl system_report = report_dir / 'pylint.report' if clean or not system_report.exists(): sh( - "pylint {flags} --output-format=parseable {apps} " + u"pylint {flags} --output-format=parseable {apps} " "> {report_dir}/pylint.report".format( flags=" ".join(flags), apps=apps_list, @@ -190,7 +191,7 @@ def run_pylint(options): if num_violations < lower_violations_limit > -1: fail_quality( result_name, - "FAILURE: Too few pylint violations. " + u"FAILURE: Too few pylint violations. " "Expected to see at least {lower_limit} pylint violations. " "Either pylint is not running correctly -or- " "the limits should be lowered and/or the lower limit should be removed.".format( @@ -202,7 +203,7 @@ def run_pylint(options): if num_violations > upper_violations_limit > -1: fail_quality( result_name, - "FAILURE: Too many pylint violations. " + u"FAILURE: Too many pylint violations. " "The limit is {upper_limit}.".format(upper_limit=upper_violations_limit) ) else: @@ -264,7 +265,7 @@ def _get_pep8_violations(clean=True): Env.METRICS_DIR.makedirs_p() if not report.exists(): - sh('pycodestyle . | tee {} -a'.format(report)) + sh(u'pycodestyle . | tee {} -a'.format(report)) violations_list = _pep8_violations(report) @@ -294,7 +295,7 @@ def run_pep8(options): # pylint: disable=unused-argument violations_list = ''.join(violations_list) # Print number of violations to log - violations_count_str = "Number of PEP 8 violations: {count}".format(count=count) + violations_count_str = u"Number of PEP 8 violations: {count}".format(count=count) print(violations_count_str) print(violations_list) @@ -331,7 +332,7 @@ def run_complexity(): print("--> Calculating cyclomatic complexity of python files...") try: sh( - "radon cc {system_string} --total-average > {complexity_report}".format( + u"radon cc {system_string} --total-average > {complexity_report}".format( system_string=system_string, complexity_report=complexity_report ) @@ -342,7 +343,7 @@ def run_complexity(): (Env.METRICS_DIR / "python_complexity") ) print("--> Python cyclomatic complexity report complete.") - print("radon cyclomatic complexity score: {metric}".format(metric=str(complexity_metric))) + print(u"radon cyclomatic complexity score: {metric}".format(metric=str(complexity_metric))) except BuildFailure: print("FAILURE: Unable to calculate python-only code-complexity.") @@ -369,7 +370,7 @@ def run_eslint(options): violations_limit = int(getattr(options, 'limit', -1)) sh( - "nodejs --max_old_space_size=4096 node_modules/.bin/eslint " + u"nodejs --max_old_space_size=4096 node_modules/.bin/eslint " "--ext .js --ext .jsx --format=compact . | tee {eslint_report}".format( eslint_report=eslint_report ), @@ -381,7 +382,7 @@ def run_eslint(options): except TypeError: fail_quality( 'eslint', - "FAILURE: Number of eslint violations could not be found in {eslint_report}".format( + u"FAILURE: Number of eslint violations could not be found in {eslint_report}".format( eslint_report=eslint_report ) ) @@ -393,7 +394,7 @@ def run_eslint(options): if num_violations > violations_limit > -1: fail_quality( 'eslint', - "FAILURE: Too many eslint violations ({count}).\nThe limit is {violations_limit}.".format( + u"FAILURE: Too many eslint violations ({count}).\nThe limit is {violations_limit}.".format( count=num_violations, violations_limit=violations_limit ) ) @@ -411,7 +412,7 @@ def _get_stylelint_violations(): formatter = 'node_modules/stylelint-formatter-pretty' sh( - "stylelint **/*.scss --custom-formatter={formatter} | tee {stylelint_report}".format( + u"stylelint **/*.scss --custom-formatter={formatter} | tee {stylelint_report}".format( formatter=formatter, stylelint_report=stylelint_report, ), @@ -423,7 +424,7 @@ def _get_stylelint_violations(): except TypeError: fail_quality( 'stylelint', - "FAILURE: Number of stylelint violations could not be found in {stylelint_report}".format( + u"FAILURE: Number of stylelint violations could not be found in {stylelint_report}".format( stylelint_report=stylelint_report ) ) @@ -450,7 +451,7 @@ def run_stylelint(options): if num_violations > violations_limit > -1: fail_quality( 'stylelint', - "FAILURE: Stylelint failed with too many violations: ({count}).\nThe limit is {violations_limit}.".format( + u"FAILURE: Stylelint failed with too many violations: ({count}).\nThe limit is {violations_limit}.".format( count=num_violations, violations_limit=violations_limit, ) @@ -493,7 +494,7 @@ def run_xsslint(options): _prepare_report_dir(xsslint_report_dir) sh( - "{repo_root}/scripts/xsslint/{xsslint_script} --rule-totals --config={cfg_module} >> {xsslint_report}".format( + u"{repo_root}/scripts/xsslint/{xsslint_script} --rule-totals --config={cfg_module} >> {xsslint_report}".format( repo_root=Env.REPO_ROOT, xsslint_script=xsslint_script, xsslint_report=xsslint_report, @@ -505,21 +506,21 @@ def run_xsslint(options): xsslint_counts = _get_xsslint_counts(xsslint_report) try: - metrics_str = "Number of {xsslint_script} violations: {num_violations}\n".format( + metrics_str = u"Number of {xsslint_script} violations: {num_violations}\n".format( xsslint_script=xsslint_script, num_violations=int(xsslint_counts['total']) ) if 'rules' in xsslint_counts and any(xsslint_counts['rules']): metrics_str += "\n" rule_keys = sorted(xsslint_counts['rules'].keys()) for rule in rule_keys: - metrics_str += "{rule} violations: {count}\n".format( + metrics_str += u"{rule} violations: {count}\n".format( rule=rule, count=int(xsslint_counts['rules'][rule]) ) except TypeError: fail_quality( 'xsslint', - "FAILURE: Number of {xsslint_script} violations could not be found in {xsslint_report}".format( + u"FAILURE: Number of {xsslint_script} violations could not be found in {xsslint_report}".format( xsslint_script=xsslint_script, xsslint_report=xsslint_report ) ) @@ -528,14 +529,14 @@ def run_xsslint(options): # Record the metric _write_metric(metrics_str, metrics_report) # Print number of violations to log. - sh("cat {metrics_report}".format(metrics_report=metrics_report), ignore_error=True) + sh(u"cat {metrics_report}".format(metrics_report=metrics_report), ignore_error=True) error_message = "" # Test total violations against threshold. if 'total' in violation_thresholds.keys(): if violation_thresholds['total'] < xsslint_counts['total']: - error_message = "Too many violations total ({count}).\nThe limit is {violations_limit}.".format( + error_message = u"Too many violations total ({count}).\nThe limit is {violations_limit}.".format( count=xsslint_counts['total'], violations_limit=violation_thresholds['total'] ) @@ -545,14 +546,14 @@ def run_xsslint(options): for threshold_key in threshold_keys: if threshold_key not in xsslint_counts['rules']: error_message += ( - "\nNumber of {xsslint_script} violations for {rule} could not be found in " + u"\nNumber of {xsslint_script} violations for {rule} could not be found in " "{xsslint_report}." ).format( xsslint_script=xsslint_script, rule=threshold_key, xsslint_report=xsslint_report ) elif violation_thresholds['rules'][threshold_key] < xsslint_counts['rules'][threshold_key]: error_message += \ - "\nToo many {rule} violations ({count}).\nThe {rule} limit is {violations_limit}.".format( + u"\nToo many {rule} violations ({count}).\nThe {rule} limit is {violations_limit}.".format( rule=threshold_key, count=xsslint_counts['rules'][threshold_key], violations_limit=violation_thresholds['rules'][threshold_key], ) @@ -560,7 +561,7 @@ def run_xsslint(options): if error_message: fail_quality( 'xsslint', - "FAILURE: XSSLinter Failed.\n{error_message}\n" + u"FAILURE: XSSLinter Failed.\n{error_message}\n" "See {xsslint_report} or run the following command to hone in on the problem:\n" " ./scripts/xss-commit-linter.sh -h".format( error_message=error_message, xsslint_report=xsslint_report @@ -583,7 +584,7 @@ def run_xsscommitlint(): _prepare_report_dir(xsscommitlint_report_dir) sh( - "{repo_root}/scripts/{xsscommitlint_script} | tee {xsscommitlint_report}".format( + u"{repo_root}/scripts/{xsscommitlint_script} | tee {xsscommitlint_report}".format( repo_root=Env.REPO_ROOT, xsscommitlint_script=xsscommitlint_script, xsscommitlint_report=xsscommitlint_report, @@ -598,13 +599,13 @@ def run_xsscommitlint(): except TypeError: fail_quality( 'xsscommitlint', - "FAILURE: Number of {xsscommitlint_script} violations could not be found in {xsscommitlint_report}".format( + u"FAILURE: Number of {xsscommitlint_script} violations could not be found in {xsscommitlint_report}".format( xsscommitlint_script=xsscommitlint_script, xsscommitlint_report=xsscommitlint_report ) ) # Print number of violations to log. - violations_count_str = "Number of {xsscommitlint_script} violations: {num_violations}\n".format( + violations_count_str = u"Number of {xsscommitlint_script} violations: {num_violations}\n".format( xsscommitlint_script=xsscommitlint_script, num_violations=num_violations ) @@ -612,10 +613,10 @@ def run_xsscommitlint(): metrics_report = (Env.METRICS_DIR / "xsscommitlint") _write_metric(violations_count_str, metrics_report) # Output report to console. - sh("cat {metrics_report}".format(metrics_report=metrics_report), ignore_error=True) + sh(u"cat {metrics_report}".format(metrics_report=metrics_report), ignore_error=True) if num_violations: fail_quality( - 'xsscommitlint', + u'xsscommitlint', "FAILURE: XSSCommitLinter Failed.\n{error_message}\n" "See {xsscommitlint_report} or run the following command to hone in on the problem:\n" " ./scripts/xss-commit-linter.sh -h".format( @@ -669,7 +670,7 @@ def _get_report_contents(filename, report_name, last_line_only=False): else: return report_file.read() else: - file_not_found_message = "FAILURE: The following log file could not be found: {file}".format(file=filename) + file_not_found_message = u"FAILURE: The following log file could not be found: {file}".format(file=filename) fail_quality(report_name, file_not_found_message) @@ -788,20 +789,20 @@ def run_quality(options): if is_html: lines = ['\n'] sep = '-------------
\n' - title = HTML("

Quality Report: {}

\n").format(linter) + title = HTML(u"

Quality Report: {}

\n").format(linter) violations_bullets = ''.join( [HTML('
  • {violation}

    • \n').format(violation=violation) for violation in violations_list] ) violations_str = HTML('
      \n{bullets}
    \n').format(bullets=HTML(violations_bullets)) - violations_count_str = HTML("Violations: {count}
    \n") - fail_line = HTML("FAILURE: {} count should be 0
    \n").format(linter) + violations_count_str = HTML(u"Violations: {count}
    \n") + fail_line = HTML(u"FAILURE: {} count should be 0
    \n").format(linter) else: lines = [] sep = '-------------\n' - title = "Quality Report: {}\n".format(linter) + title = u"Quality Report: {}\n".format(linter) violations_str = ''.join(violations_list) - violations_count_str = "Violations: {count}\n" - fail_line = "FAILURE: {} count should be {}\n".format(linter, limit) + violations_count_str = u"Violations: {count}\n" + fail_line = u"FAILURE: {} count should be {}\n".format(linter, limit) violations_count_str = violations_count_str.format(count=count) @@ -829,7 +830,7 @@ def run_quality(options): # ----- Set up for diff-quality pylint call ----- # Set the string to be used for the diff-quality --compare-branch switch. compare_branch = getattr(options, 'compare_branch', u'origin/master') - compare_commit = sh('git merge-base HEAD {}'.format(compare_branch), capture=True).strip() + compare_commit = sh(u'git merge-base HEAD {}'.format(compare_branch), capture=True).strip() if sh('git rev-parse HEAD', capture=True).strip() != compare_commit: compare_branch_string = u'--compare-branch={0}'.format(compare_commit) @@ -881,7 +882,7 @@ def run_diff_quality( """ try: sh( - "diff-quality --violations={type} " + u"diff-quality --violations={type} " "{reports} {percentage_string} {compare_branch_string} " "--html-report {dquality_dir}/diff_quality_{type}.html ".format( type=violations_type, diff --git a/pavelib/servers.py b/pavelib/servers.py index 2ba99a3acf..f7f36a9f30 100644 --- a/pavelib/servers.py +++ b/pavelib/servers.py @@ -255,7 +255,7 @@ def update_db(options): fake = "--fake-initial" if getattr(options, 'fake_initial', False) else "" for system in ('lms', 'cms'): # pylint: disable=line-too-long - sh("NO_EDXAPP_SUDO=1 EDX_PLATFORM_SETTINGS_OVERRIDE={settings} /edx/bin/edxapp-migrate-{system} --traceback --pythonpath=. {fake}".format( + sh(u"NO_EDXAPP_SUDO=1 EDX_PLATFORM_SETTINGS_OVERRIDE={settings} /edx/bin/edxapp-migrate-{system} --traceback --pythonpath=. {fake}".format( settings=settings, system=system, fake=fake)) @@ -278,9 +278,9 @@ def check_settings(args): settings = args.settings[0] try: - import_cmd = "echo 'import {system}.envs.{settings}'".format(system=system, settings=settings) + import_cmd = u"echo 'import {system}.envs.{settings}'".format(system=system, settings=settings) django_shell_cmd = django_cmd(system, settings, 'shell', '--plain', '--pythonpath=.') - sh("{import_cmd} | {shell_cmd}".format(import_cmd=import_cmd, shell_cmd=django_shell_cmd)) + sh(u"{import_cmd} | {shell_cmd}".format(import_cmd=import_cmd, shell_cmd=django_shell_cmd)) except: # pylint: disable=bare-except print("Failed to import settings", file=sys.stderr) diff --git a/pavelib/tests.py b/pavelib/tests.py index e4aa17fa82..eafd6aa3b2 100644 --- a/pavelib/tests.py +++ b/pavelib/tests.py @@ -294,7 +294,7 @@ def coverage(): if not (report_dir / '.coverage').isfile(): # This may be that the coverage files were generated using -p, # try to combine them to the one file that we need. - sh("coverage combine --rcfile={}".format(rcfile)) + sh(u"coverage combine --rcfile={}".format(rcfile)) if not os.path.getsize(report_dir / '.coverage') > 50: # Check if the .coverage data file is larger than the base file, @@ -309,9 +309,9 @@ def coverage(): return # Generate the coverage.py XML report - sh("coverage xml --rcfile={}".format(rcfile)) + sh(u"coverage xml --rcfile={}".format(rcfile)) # Generate the coverage.py HTML report - sh("coverage html --rcfile={}".format(rcfile)) + sh(u"coverage html --rcfile={}".format(rcfile)) diff_coverage() # pylint: disable=no-value-for-parameter @@ -347,7 +347,7 @@ def diff_coverage(options): # Generate the diff coverage reports (HTML and console) sh( - "diff-cover {xml_report_str} --compare-branch={compare_branch} " + u"diff-cover {xml_report_str} --compare-branch={compare_branch} " "--html-report {diff_html_path}".format( xml_report_str=xml_report_str, compare_branch=compare_branch, diff --git a/pavelib/utils/db_utils.py b/pavelib/utils/db_utils.py index 182e12e1c2..a6f877aaaa 100644 --- a/pavelib/utils/db_utils.py +++ b/pavelib/utils/db_utils.py @@ -24,9 +24,9 @@ def remove_files_from_folder(files, folder): file_with_path = os.path.join(folder, file_name) try: os.remove(file_with_path) - print('\tRemoved {}'.format(file_with_path)) + print(u'\tRemoved {}'.format(file_with_path)) except OSError: - print('\tCould not remove {}. Continuing.'.format(file_with_path)) + print(u'\tCould not remove {}. Continuing.'.format(file_with_path)) continue @@ -40,9 +40,9 @@ def reset_test_db(db_cache_files, update_cache_files=True, use_existing_db=False """ cmd = '{}/scripts/reset-test-db.sh'.format(Env.REPO_ROOT) if update_cache_files: - cmd = '{} --rebuild_cache'.format(cmd) + cmd = u'{} --rebuild_cache'.format(cmd) if use_existing_db: - cmd = '{} --use-existing-db'.format(cmd) + cmd = u'{} --use-existing-db'.format(cmd) sh(cmd) verify_files_exist(db_cache_files) @@ -77,7 +77,7 @@ def fingerprint_bokchoy_db_files(migration_output_files, all_db_files): msg = "Computing the fingerprint." print(msg) fingerprint = compute_fingerprint(file_paths) - print("The fingerprint for bokchoy db files is: {}".format(fingerprint)) + print(u"The fingerprint for bokchoy db files is: {}".format(fingerprint)) return fingerprint @@ -100,7 +100,7 @@ def verify_files_exist(files): for file_name in files: file_path = os.path.join(CACHE_FOLDER, file_name) if not os.path.isfile(file_path): - msg = "Did not find expected file: {}".format(file_path) + msg = u"Did not find expected file: {}".format(file_path) raise BuildFailure(msg) @@ -112,7 +112,7 @@ def calculate_bokchoy_migrations(migration_output_files): NOTE: the script first clears out the database, then calculates what migrations need to be run, which is all of them. """ - sh('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)) + sh(u'{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)) verify_files_exist(migration_output_files) @@ -136,7 +136,7 @@ def is_fingerprint_in_bucket(fingerprint, bucket_name): conn = boto.connect_s3(anon=True) bucket = conn.get_bucket(bucket_name) except Exception as e: # pylint: disable=broad-except - print("Exception caught trying to reach S3 bucket {}: {}".format(bucket_name, e)) + print(u"Exception caught trying to reach S3 bucket {}: {}".format(bucket_name, e)) return False key = boto.s3.key.Key(bucket=bucket, name=zipfile_name) return key.exists() @@ -158,12 +158,12 @@ def get_file_from_s3(bucket_name, zipfile_name, path): """ Get the file from s3 and save it to disk. """ - print ("Retrieving {} from bucket {}.".format(zipfile_name, bucket_name)) + print (u"Retrieving {} from bucket {}.".format(zipfile_name, bucket_name)) conn = boto.connect_s3(anon=True) bucket = conn.get_bucket(bucket_name) key = boto.s3.key.Key(bucket=bucket, name=zipfile_name) if not key.exists(): - msg = "Did not find expected file {} in the S3 bucket {}".format( + msg = u"Did not find expected file {} in the S3 bucket {}".format( zipfile_name, bucket_name ) raise BuildFailure(msg) @@ -214,7 +214,7 @@ def upload_to_s3(file_name, file_path, bucket_name): """ Upload the specified files to an s3 bucket. """ - print("Uploading {} to s3 bucket {}".format(file_name, bucket_name)) + print(u"Uploading {} to s3 bucket {}".format(file_name, bucket_name)) try: conn = boto.connect_s3() except boto.exception.NoAuthHandlerFound: @@ -230,9 +230,9 @@ def upload_to_s3(file_name, file_path, bucket_name): key = boto.s3.key.Key(bucket=bucket, name=file_name) bytes_written = key.set_contents_from_filename(file_path, replace=False, policy='public-read') if bytes_written: - msg = "Wrote {} bytes to {}.".format(bytes_written, key.name) + msg = u"Wrote {} bytes to {}.".format(bytes_written, key.name) else: - msg = "File {} already existed in bucket {}.".format(key.name, bucket_name) + msg = u"File {} already existed in bucket {}.".format(key.name, bucket_name) print(msg) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index 817eee76bb..e761b284de 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -35,7 +35,7 @@ def repo_root(): absolute_path = file_path.abspath() break except OSError: - print('Attempt {}/180 to get an absolute path failed'.format(attempt)) + print(u'Attempt {}/180 to get an absolute path failed'.format(attempt)) if attempt < 180: attempt += 1 sleep(1) @@ -251,7 +251,7 @@ class Env(object): django_cmd( system, settings, - "print_setting {django_setting} 2>{log_file}".format( + u"print_setting {django_setting} 2>{log_file}".format( django_setting=django_setting, log_file=cls.PRINT_SETTINGS_LOG_FILE ) @@ -260,7 +260,7 @@ class Env(object): ) return unicode(value).strip() except BuildFailure: - print("Unable to print the value of the {} setting:".format(django_setting)) + print(u"Unable to print the value of the {} setting:".format(django_setting)) with io.open(cls.PRINT_SETTINGS_LOG_FILE, 'r') as f: print(f.read()) sys.exit(1) @@ -300,8 +300,8 @@ class Env(object): env_path = env_path.parent.parent / env_path.basename() if not env_path.isfile(): print( - "Warning: could not find environment JSON file " - "at '{path}'".format(path=env_path), + u"Warning: could not find environment JSON file " + "at '{path}'".format(path=env_path), # pylint: disable=unicode-format-string file=sys.stderr, ) return dict() @@ -313,8 +313,8 @@ class Env(object): except ValueError: print( - "Error: Could not parse JSON " - "in {path}".format(path=env_path), + u"Error: Could not parse JSON " + "in {path}".format(path=env_path), # pylint: disable=unicode-format-string file=sys.stderr, ) sys.exit(1) diff --git a/pavelib/utils/passthrough_opts.py b/pavelib/utils/passthrough_opts.py index cff98f46c3..311f0556b0 100644 --- a/pavelib/utils/passthrough_opts.py +++ b/pavelib/utils/passthrough_opts.py @@ -58,9 +58,9 @@ class PassthroughOptionParser(OptionParser): if len(rargs) < nargs: if nargs == 1: - self.error("%s option requires an argument" % opt) + self.error(u"%s option requires an argument" % opt) else: - self.error("%s option requires %d arguments" + self.error(u"%s option requires %d arguments" % (opt, nargs)) elif nargs == 1: value = rargs.pop(0) @@ -69,7 +69,7 @@ class PassthroughOptionParser(OptionParser): del rargs[0:nargs] elif had_explicit_value: - self.error("%s option does not take a value" % opt) + self.error(u"%s option does not take a value" % opt) else: value = None @@ -103,9 +103,9 @@ class PassthroughOptionParser(OptionParser): nargs = option.nargs if len(rargs) < nargs: if nargs == 1: - self.error("%s option requires an argument" % opt) + self.error(u"%s option requires an argument" % opt) else: - self.error("%s option requires %d arguments" + self.error(u"%s option requires %d arguments" % (opt, nargs)) elif nargs == 1: diff --git a/pavelib/utils/process.py b/pavelib/utils/process.py index 5cc8254b29..24b49f525f 100644 --- a/pavelib/utils/process.py +++ b/pavelib/utils/process.py @@ -69,7 +69,7 @@ def run_multi_processes(cmd_list, out_log=None, err_log=None): # pylint: disable=broad-except except Exception as err: - print("Error running process {}".format(err), file=sys.stderr) + print(u"Error running process {}".format(err), file=sys.stderr) finally: for pid in pids: diff --git a/pavelib/utils/test/bokchoy_utils.py b/pavelib/utils/test/bokchoy_utils.py index 7e478db640..4796464de9 100644 --- a/pavelib/utils/test/bokchoy_utils.py +++ b/pavelib/utils/test/bokchoy_utils.py @@ -42,13 +42,13 @@ def start_servers(options): for service, info in Env.BOK_CHOY_SERVERS.iteritems(): address = "0.0.0.0:{}".format(info['port']) - cmd = ("DEFAULT_STORE={default_store} ").format(default_store=options.default_store) + cmd = (u"DEFAULT_STORE={default_store} ").format(default_store=options.default_store) if coveragerc: - cmd += ("coverage run --rcfile={coveragerc} -m ").format(coveragerc=coveragerc) + cmd += (u"coverage run --rcfile={coveragerc} -m ").format(coveragerc=coveragerc) else: cmd += "python -m " cmd += ( - "manage {service} --settings {settings} runserver " + u"manage {service} --settings {settings} runserver " "{address} --traceback --noreload".format( service=service, settings=Env.SETTINGS, @@ -59,7 +59,7 @@ def start_servers(options): for service, info in Env.BOK_CHOY_STUBS.iteritems(): cmd = ( - "python -m stubs.start {service} {port} " + u"python -m stubs.start {service} {port} " "{config}".format( service=service, port=info['port'], @@ -74,7 +74,7 @@ def wait_for_server(server, port): Wait for a server to respond with status 200 """ print( - "Checking server {server} on port {port}".format( + u"Checking server {server} on port {port}".format( server=server, port=port, ) @@ -114,7 +114,7 @@ def wait_for_test_servers(): if not ready: msg = colorize( "red", - "Could not contact {} test server".format(service) + u"Could not contact {} test server".format(service) ) print(msg) sys.exit(1) @@ -126,7 +126,7 @@ def is_mongo_running(): """ # The mongo command will connect to the service, # failing with a non-zero exit code if it cannot connect. - output = os.popen('mongo --host {} --eval "print(\'running\')"'.format(Env.MONGO_HOST)).read() + output = os.popen(u'mongo --host {} --eval "print(\'running\')"'.format(Env.MONGO_HOST)).read() return output and "running" in output @@ -158,7 +158,7 @@ def clear_mongo(): Clears mongo database. """ sh( - "mongo --host {} {} --eval 'db.dropDatabase()' > /dev/null".format( + u"mongo --host {} {} --eval 'db.dropDatabase()' > /dev/null".format( Env.MONGO_HOST, Env.BOK_CHOY_MONGO_DATABASE, ) diff --git a/pavelib/utils/test/suites/acceptance_suite.py b/pavelib/utils/test/suites/acceptance_suite.py index 9532df33b5..f1029dbec6 100644 --- a/pavelib/utils/test/suites/acceptance_suite.py +++ b/pavelib/utils/test/suites/acceptance_suite.py @@ -49,22 +49,22 @@ def setup_acceptance_db(): # Copy the cached database to the test root directory for db_alias in DBS: - sh("cp {db_cache} {db}".format(db_cache=DB_CACHES[db_alias], db=DBS[db_alias])) + sh(u"cp {db_cache} {db}".format(db_cache=DB_CACHES[db_alias], db=DBS[db_alias])) # Run migrations to update the db, starting from its cached state for db_alias in sorted(DBS): # pylint: disable=line-too-long - sh("./manage.py lms --settings {} migrate --traceback --noinput --fake-initial --database {}".format(settings, db_alias)) - sh("./manage.py cms --settings {} migrate --traceback --noinput --fake-initial --database {}".format(settings, db_alias)) + sh(u"./manage.py lms --settings {} migrate --traceback --noinput --fake-initial --database {}".format(settings, db_alias)) + sh(u"./manage.py cms --settings {} migrate --traceback --noinput --fake-initial --database {}".format(settings, db_alias)) else: # If no cached database exists, migrate then create the cache for db_alias in sorted(DBS.keys()): - sh("./manage.py lms --settings {} migrate --traceback --noinput --database {}".format(settings, db_alias)) - sh("./manage.py cms --settings {} migrate --traceback --noinput --database {}".format(settings, db_alias)) + sh(u"./manage.py lms --settings {} migrate --traceback --noinput --database {}".format(settings, db_alias)) + sh(u"./manage.py cms --settings {} migrate --traceback --noinput --database {}".format(settings, db_alias)) # Create the cache if it doesn't already exist for db_alias in DBS.keys(): - sh("cp {db} {db_cache}".format(db_cache=DB_CACHES[db_alias], db=DBS[db_alias])) + sh(u"cp {db} {db_cache}".format(db_cache=DB_CACHES[db_alias], db=DBS[db_alias])) class AcceptanceTest(TestSuite): @@ -95,7 +95,7 @@ class AcceptanceTest(TestSuite): lettuce_host = ['LETTUCE_HOST={}'.format(Env.SERVER_HOST)] if Env.USING_DOCKER else [] report_file = self.report_dir / "{}.xml".format(self.system) - report_args = ["--xunit-file {}".format(report_file)] + report_args = [u"--xunit-file {}".format(report_file)] return lettuce_host + [ # set DBUS_SESSION_BUS_ADDRESS to avoid hangs on Chrome "DBUS_SESSION_BUS_ADDRESS=/dev/null", @@ -150,7 +150,7 @@ class AcceptanceTestSuite(TestSuite): for default_store in stores: kwargs['system'] = system kwargs['default_store'] = default_store - self.subsuites.append(AcceptanceTest('{} acceptance using {}'.format(system, default_store), **kwargs)) + self.subsuites.append(AcceptanceTest(u'{} acceptance using {}'.format(system, default_store), **kwargs)) def __enter__(self): super(AcceptanceTestSuite, self).__enter__() diff --git a/pavelib/utils/test/suites/bokchoy_suite.py b/pavelib/utils/test/suites/bokchoy_suite.py index a44de01710..287ac5d143 100644 --- a/pavelib/utils/test/suites/bokchoy_suite.py +++ b/pavelib/utils/test/suites/bokchoy_suite.py @@ -1,3 +1,4 @@ +# pylint: disable=unicode-format-string """ Class used for defining and running Bok Choy acceptance test suite """ @@ -48,7 +49,7 @@ def load_bok_choy_data(options): """ print('Loading data from json fixtures in db_fixtures directory') sh( - "DEFAULT_STORE={default_store}" + u"DEFAULT_STORE={default_store}" " ./manage.py lms --settings {settings} loaddata --traceback" " common/test/db_fixtures/*.json".format( default_store=options.default_store, @@ -73,11 +74,11 @@ def load_courses(options): `test_root/courses/`. """ if 'imports_dir' in options: - msg = colorize('green', "Importing courses from {}...".format(options.imports_dir)) + msg = colorize('green', u"Importing courses from {}...".format(options.imports_dir)) print(msg) sh( - "DEFAULT_STORE={default_store}" + u"DEFAULT_STORE={default_store}" " ./manage.py cms --settings={settings} import {import_dir}".format( default_store=options.default_store, import_dir=options.imports_dir, @@ -100,7 +101,7 @@ def update_fixtures(): print(msg) sh( - " ./manage.py lms --settings={settings} update_fixtures".format( + u" ./manage.py lms --settings={settings} update_fixtures".format( settings=Env.SETTINGS ) ) @@ -132,7 +133,7 @@ def get_test_course(options): print(msg) sh( - 'wget {tar_gz_file} -O {zipped_course}'.format( + u'wget {tar_gz_file} -O {zipped_course}'.format( tar_gz_file=DEMO_COURSE_TAR_GZ, zipped_course=zipped_course, ) @@ -142,7 +143,7 @@ def get_test_course(options): print(msg) sh( - 'tar zxf {zipped_course} -C {courses_dir}'.format( + u'tar zxf {zipped_course} -C {courses_dir}'.format( zipped_course=zipped_course, courses_dir=options.imports_dir, ) @@ -287,7 +288,7 @@ class BokChoyTestSuite(TestSuite): # Clean up data we created in the databases msg = colorize('green', "Cleaning up databases...") print(msg) - sh("./manage.py lms --settings {settings} flush --traceback --noinput".format(settings=Env.SETTINGS)) + sh(u"./manage.py lms --settings {settings} flush --traceback --noinput".format(settings=Env.SETTINGS)) clear_mongo() @property @@ -300,7 +301,7 @@ class BokChoyTestSuite(TestSuite): if self.num_processes != 1: # Construct "multiprocess" pytest command command += [ - "-n {}".format(self.num_processes), + u"-n {}".format(self.num_processes), "--color=no", ] if self.verbosity < 1: @@ -308,7 +309,7 @@ class BokChoyTestSuite(TestSuite): elif self.verbosity > 1: command.append("--verbose") if self.eval_attr: - command.append("-a '{}'".format(self.eval_attr)) + command.append(u"-a '{}'".format(self.eval_attr)) return command diff --git a/pavelib/utils/test/suites/pytest_suite.py b/pavelib/utils/test/suites/pytest_suite.py index 5fb755bc90..4e21de11b6 100644 --- a/pavelib/utils/test/suites/pytest_suite.py +++ b/pavelib/utils/test/suites/pytest_suite.py @@ -1,3 +1,4 @@ +# pylint: disable=unicode-format-string """ Classes used for defining and running pytest test suites """ @@ -161,27 +162,27 @@ class SystemTestSuite(PytestSuite): xdist_remote_processes = self.processes for ip in self.xdist_ip_addresses.split(','): # The django settings runtime command does not propagate to xdist remote workers - django_env_var_cmd = 'export DJANGO_SETTINGS_MODULE={}' \ + django_env_var_cmd = u'export DJANGO_SETTINGS_MODULE={}' \ .format('{}.envs.{}'.format(self.root, self.settings)) - xdist_string = '--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \ + xdist_string = u'--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \ '//python="source /edx/app/edxapp/edxapp_env; {}; python"' \ '//chdir="/edx/app/edxapp/edx-platform"' \ .format(xdist_remote_processes, ip, django_env_var_cmd) cmd.append(xdist_string) for rsync_dir in Env.rsync_dirs(): - cmd.append('--rsyncdir {}'.format(rsync_dir)) + cmd.append(u'--rsyncdir {}'.format(rsync_dir)) else: if self.processes == -1: cmd.append('-n auto') cmd.append('--dist=loadscope') elif self.processes != 0: - cmd.append('-n {}'.format(self.processes)) + cmd.append(u'-n {}'.format(self.processes)) cmd.append('--dist=loadscope') if not self.randomize: cmd.append('-p no:randomly') if self.eval_attr: - cmd.append("-a '{}'".format(self.eval_attr)) + cmd.append(u"-a '{}'".format(self.eval_attr)) cmd.extend(self.passthrough_options) cmd.append(self.test_id) @@ -284,25 +285,25 @@ class LibTestSuite(PytestSuite): django_env_var_cmd = "export DJANGO_SETTINGS_MODULE='lms.envs.test'" else: django_env_var_cmd = "export DJANGO_SETTINGS_MODULE='openedx.tests.settings'" - xdist_string = '--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \ + xdist_string = u'--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \ '//python="source /edx/app/edxapp/edxapp_env; {}; python"' \ '//chdir="/edx/app/edxapp/edx-platform"' \ .format(xdist_remote_processes, ip, django_env_var_cmd) cmd.append(xdist_string) for rsync_dir in Env.rsync_dirs(): - cmd.append('--rsyncdir {}'.format(rsync_dir)) + cmd.append(u'--rsyncdir {}'.format(rsync_dir)) else: if self.processes == -1: cmd.append('-n auto') cmd.append('--dist=loadscope') elif self.processes != 0: - cmd.append('-n {}'.format(self.processes)) + cmd.append(u'-n {}'.format(self.processes)) cmd.append('--dist=loadscope') if not self.randomize: cmd.append("-p no:randomly") if self.eval_attr: - cmd.append("-a '{}'".format(self.eval_attr)) + cmd.append(u"-a '{}'".format(self.eval_attr)) cmd.append(self.test_id) diff --git a/pavelib/utils/test/suites/suite.py b/pavelib/utils/test/suites/suite.py index 83be219e04..bd6bcd9cee 100644 --- a/pavelib/utils/test/suites/suite.py +++ b/pavelib/utils/test/suites/suite.py @@ -1,3 +1,4 @@ +# pylint: disable=unicode-format-string """ A class used for defining and running test suites """ @@ -39,7 +40,7 @@ class TestSuite(object): i.e. Checking for and defining required directories. """ - print("\nSetting up for {suite_name}".format(suite_name=self.root)) + print(u"\nSetting up for {suite_name}".format(suite_name=self.root)) self.failed_suites = [] def __exit__(self, exc_type, exc_value, traceback): @@ -52,7 +53,7 @@ class TestSuite(object): i.e. Cleaning mongo after the lms tests run. """ - print("\nCleaning up after {suite_name}".format(suite_name=self.root)) + print(u"\nCleaning up after {suite_name}".format(suite_name=self.root)) @property def cmd(self): @@ -123,10 +124,10 @@ class TestSuite(object): Writes a list of failed_suites to sys.stderr """ if self.failed_suites: - msg = colorize('red', "\n\n{bar}\nTests failed in the following suites:\n* ".format(bar="=" * 48)) + msg = colorize('red', u"\n\n{bar}\nTests failed in the following suites:\n* ".format(bar="=" * 48)) msg += colorize('red', '\n* '.join([s.root for s in self.failed_suites]) + '\n\n') else: - msg = colorize('green', "\n\n{bar}\nNo test failures ".format(bar="=" * 48)) + msg = colorize('green', u"\n\n{bar}\nNo test failures ".format(bar="=" * 48)) print(msg) diff --git a/pavelib/utils/test/utils.py b/pavelib/utils/test/utils.py index 8743ca8187..f9e8eda4e6 100644 --- a/pavelib/utils/test/utils.py +++ b/pavelib/utils/test/utils.py @@ -32,7 +32,7 @@ def clean_test_files(): # This find command removes all the *.pyc files that aren't in the .git # directory. See this blog post for more details: # http://nedbatchelder.com/blog/201505/be_careful_deleting_files_around_git.html - sh(r"find . -name '.git' -prune -o -name '*.pyc' -exec rm {} \;") + sh(r"find . -name '.git' -prune -o -name '*.pyc' -exec rm {} \;") # pylint: disable=unicode-format-string sh("rm -rf test_root/log/auto_screenshots/*") sh("rm -rf /tmp/mako_[cl]ms") @@ -55,7 +55,7 @@ def clean_dir(directory): """ # We delete the files but preserve the directory structure # so that coverage.py has a place to put the reports. - sh('find {dir} -type f -delete'.format(dir=directory)) + sh(u'find {dir} -type f -delete'.format(dir=directory)) @task @@ -84,7 +84,7 @@ def clean_mongo(): """ Clean mongo test databases """ - sh("mongo {host}:{port} {repo_root}/scripts/delete-mongo-test-dbs.js".format( + sh(u"mongo {host}:{port} {repo_root}/scripts/delete-mongo-test-dbs.js".format( host=Env.MONGO_HOST, port=MONGO_PORT_NUM, repo_root=Env.REPO_ROOT, @@ -111,10 +111,10 @@ def check_firefox_version(): driver.close() if firefox_ver < MINIMUM_FIREFOX_VERSION: raise Exception( - 'Required firefox version not found.\n' - 'Expected: {expected_version}; Actual: {actual_version}.\n\n' - 'Make sure that the edx.devstack.firefox container is up-to-date and running\n' - '\t{expected_version}'.format( + u'Required firefox version not found.\n' + u'Expected: {expected_version}; Actual: {actual_version}.\n\n' + u'Make sure that the edx.devstack.firefox container is up-to-date and running\n' + u'\t{expected_version}'.format( actual_version=firefox_ver, expected_version=MINIMUM_FIREFOX_VERSION ) @@ -135,15 +135,15 @@ def check_firefox_version(): if firefox_ver < MINIMUM_FIREFOX_VERSION: raise Exception( - 'Required firefox version not found.\n' - 'Expected: {expected_version}; Actual: {actual_version}.\n\n' - 'As the vagrant user in devstack, run the following:\n\n' - '\t$ sudo wget -O /tmp/firefox_42.deb {debian_path}\n' - '\t$ sudo apt-get remove firefox\n\n' - '\t$ sudo gdebi -nq /tmp/firefox_42.deb\n\n' - 'Confirm the new version:\n' - '\t$ firefox --version\n' - '\t{expected_version}'.format( + u'Required firefox version not found.\n' + u'Expected: {expected_version}; Actual: {actual_version}.\n\n' + u'As the vagrant user in devstack, run the following:\n\n' + u'\t$ sudo wget -O /tmp/firefox_42.deb {debian_path}\n' + u'\t$ sudo apt-get remove firefox\n\n' + u'\t$ sudo gdebi -nq /tmp/firefox_42.deb\n\n' + u'Confirm the new version:\n' + u'\t$ firefox --version\n' + u'\t{expected_version}'.format( actual_version=firefox_ver, expected_version=expected_firefox_ver, debian_path=debian_path