Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: pyupgrade on pavelib & scripts (#26769)

Browse Source
This commit is contained in:
M. Zulqarnain
2021-03-12 14:14:53 +05:00
committed by GitHub
parent b1b3ba9c81
commit 2c44315ce0
26 changed files with 132 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
pavelib/utils/db_utils.py
View File

@@ -13,7 +13,7 @@ from pavelib.prereqs import compute_fingerprint
from pavelib.utils.envs import Env
CACHE_FOLDER = 'common/test/db_cache'
FINGERPRINT_FILEPATH = '{}/{}/bok_choy_migrations.sha1'.format(Env.REPO_ROOT, CACHE_FOLDER)
FINGERPRINT_FILEPATH = f'{Env.REPO_ROOT}/{CACHE_FOLDER}/bok_choy_migrations.sha1'
def remove_files_from_folder(files, folder):
@@ -25,9 +25,9 @@ def remove_files_from_folder(files, folder):
file_with_path = os.path.join(folder, file_name)
try:
os.remove(file_with_path)
print('\tRemoved {}'.format(file_with_path))
print(f'\tRemoved {file_with_path}')
except OSError:
print('\tCould not remove {}. Continuing.'.format(file_with_path))
print(f'\tCould not remove {file_with_path}. Continuing.')
continue
@@ -39,11 +39,11 @@ def reset_test_db(db_cache_files, update_cache_files=True, use_existing_db=False
exist), load in the db cache files files if they exist on disk,
and optionally apply migrations and write up-to-date cache files.
"""
cmd = '{}/scripts/reset-test-db.sh'.format(Env.REPO_ROOT)
cmd = f'{Env.REPO_ROOT}/scripts/reset-test-db.sh'
if update_cache_files:
cmd = '{} --rebuild_cache'.format(cmd)
cmd = f'{cmd} --rebuild_cache'
if use_existing_db:
cmd = '{} --use-existing-db'.format(cmd)
cmd = f'{cmd} --use-existing-db'
sh(cmd)
verify_files_exist(db_cache_files)
@@ -78,7 +78,7 @@ def fingerprint_bokchoy_db_files(migration_output_files, all_db_files):
msg = "Computing the fingerprint."
print(msg)
fingerprint = compute_fingerprint(file_paths)
print("The fingerprint for bokchoy db files is: {}".format(fingerprint))
print(f"The fingerprint for bokchoy db files is: {fingerprint}")
return fingerprint
@@ -101,7 +101,7 @@ def verify_files_exist(files):
for file_name in files:
file_path = os.path.join(CACHE_FOLDER, file_name)
if not os.path.isfile(file_path):
msg = "Did not find expected file: {}".format(file_path)
msg = f"Did not find expected file: {file_path}"
raise BuildFailure(msg)
@@ -113,7 +113,7 @@ def calculate_bokchoy_migrations(migration_output_files):
NOTE: the script first clears out the database, then calculates
what migrations need to be run, which is all of them.
"""
sh('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT))
sh(f'{Env.REPO_ROOT}/scripts/reset-test-db.sh --calculate_migrations')
verify_files_exist(migration_output_files)
@@ -132,12 +132,12 @@ def is_fingerprint_in_bucket(fingerprint, bucket_name):
If there is any issue reaching the bucket, show the exception but continue by
returning False
"""
zipfile_name = '{}.tar.gz'.format(fingerprint)
zipfile_name = f'{fingerprint}.tar.gz'
try:
conn = boto.connect_s3(anon=True)
bucket = conn.get_bucket(bucket_name)
except Exception as e: # pylint: disable=broad-except
print("Exception caught trying to reach S3 bucket {}: {}".format(bucket_name, e))
print(f"Exception caught trying to reach S3 bucket {bucket_name}: {e}")
return False
key = boto.s3.key.Key(bucket=bucket, name=zipfile_name)
return key.exists()
@@ -159,7 +159,7 @@ def get_file_from_s3(bucket_name, zipfile_name, path):
"""
Get the file from s3 and save it to disk.
"""
print("Retrieving {} from bucket {}.".format(zipfile_name, bucket_name))
print(f"Retrieving {zipfile_name} from bucket {bucket_name}.")
conn = boto.connect_s3(anon=True)
bucket = conn.get_bucket(bucket_name)
key = boto.s3.key.Key(bucket=bucket, name=zipfile_name)
@@ -191,7 +191,7 @@ def refresh_bokchoy_db_cache_from_s3(fingerprint, bucket_name, bokchoy_db_files)
"""
path = CACHE_FOLDER
if is_fingerprint_in_bucket(fingerprint, bucket_name):
zipfile_name = '{}.tar.gz'.format(fingerprint)
zipfile_name = f'{fingerprint}.tar.gz'
get_file_from_s3(bucket_name, zipfile_name, path)
zipfile_path = os.path.join(path, zipfile_name)
print("Extracting db cache files.")
@@ -203,7 +203,7 @@ def create_tarfile_from_db_cache(fingerprint, files, path):
"""
Create a tar.gz file with the current bokchoy DB cache files.
"""
zipfile_name = '{}.tar.gz'.format(fingerprint)
zipfile_name = f'{fingerprint}.tar.gz'
zipfile_path = os.path.join(path, zipfile_name)
with tarfile.open(name=zipfile_path, mode='w:gz') as tar_file:
for name in files:
@@ -215,7 +215,7 @@ def upload_to_s3(file_name, file_path, bucket_name, replace=False):
"""
Upload the specified files to an s3 bucket.
"""
print("Uploading {} to s3 bucket {}".format(file_name, bucket_name))
print(f"Uploading {file_name} to s3 bucket {bucket_name}")
try:
conn = boto.connect_s3()
except boto.exception.NoAuthHandlerFound:
@@ -231,9 +231,9 @@ def upload_to_s3(file_name, file_path, bucket_name, replace=False):
key = boto.s3.key.Key(bucket=bucket, name=file_name)
bytes_written = key.set_contents_from_filename(file_path, replace=replace, policy='public-read')
if bytes_written:
msg = "Wrote {} bytes to {}.".format(bytes_written, key.name)
msg = f"Wrote {bytes_written} bytes to {key.name}."
else:
msg = "File {} already existed in bucket {}.".format(key.name, bucket_name)
msg = f"File {key.name} already existed in bucket {bucket_name}."
print(msg)

15
pavelib/utils/envs.py
View File

@@ -1,7 +1,7 @@
"""
Helper functions for loading environment settings.
"""
import configparser
import json
import os
import sys
@@ -11,7 +11,6 @@ import memcache
from lazy import lazy
from path import Path as path
from paver.easy import BuildFailure, sh
from six.moves import configparser
from pavelib.utils.cmd import django_cmd
@@ -33,7 +32,7 @@ def repo_root():
absolute_path = file_path.abspath()
break
except OSError:
print('Attempt {}/180 to get an absolute path failed'.format(attempt))
print(f'Attempt {attempt}/180 to get an absolute path failed')
if attempt < 180:
attempt += 1
sleep(1)
@@ -137,7 +136,7 @@ class Env:
'video': {
'port': 8777,
'log': BOK_CHOY_LOG_DIR / "bok_choy_video_sources.log",
'config': "root_dir={}".format(VIDEO_SOURCE_DIR),
'config': f"root_dir={VIDEO_SOURCE_DIR}",
},
'youtube': {
@@ -170,7 +169,7 @@ class Env:
MONGO_HOST = 'edx.devstack.mongo' if USING_DOCKER else 'localhost'
BOK_CHOY_MONGO_DATABASE = "test"
BOK_CHOY_CACHE_HOST = 'edx.devstack.memcached' if USING_DOCKER else '0.0.0.0'
BOK_CHOY_CACHE = memcache.Client(['{}:11211'.format(BOK_CHOY_CACHE_HOST)], debug=0)
BOK_CHOY_CACHE = memcache.Client([f'{BOK_CHOY_CACHE_HOST}:11211'], debug=0)
# Test Ids Directory
TEST_DIR = REPO_ROOT / ".testids"
@@ -269,8 +268,8 @@ class Env:
# else for cases where values are not found & sh returns one None value
return tuple(str(value).splitlines()) if value else tuple(None for _ in range(settings_length))
except BuildFailure:
print("Unable to print the value of the {} setting:".format(django_settings))
with open(cls.PRINT_SETTINGS_LOG_FILE, 'r') as f:
print(f"Unable to print the value of the {django_settings} setting:")
with open(cls.PRINT_SETTINGS_LOG_FILE) as f:
print(f.read())
sys.exit(1)
@@ -315,7 +314,7 @@ class Env:
# Find the env JSON file
if self.SERVICE_VARIANT:
env_path = self.REPO_ROOT.parent / "{service}.env.json".format(service=self.SERVICE_VARIANT)
env_path = self.REPO_ROOT.parent / f"{self.SERVICE_VARIANT}.env.json"
else:
env_path = path("env.json").abspath()

2
pavelib/utils/passthrough_opts.py
View File

@@ -10,9 +10,9 @@ Provides:
from optparse import BadOptionError, OptionParser
from unittest.mock import patch
import paver.tasks
from mock import patch
class PassthroughOptionParser(OptionParser):

2
pavelib/utils/process.py
View File

@@ -69,7 +69,7 @@ def run_multi_processes(cmd_list, out_log=None, err_log=None):
# pylint: disable=broad-except
except Exception as err:
print("Error running process {}".format(err), file=sys.stderr)
print(f"Error running process {err}", file=sys.stderr)
finally:
for pid in pids:

8
pavelib/utils/test/bokchoy_utils.py
View File

@@ -7,8 +7,8 @@ import os
import subprocess
import sys
import time
from http.client import HTTPConnection
import six
from paver import tasks
from paver.easy import cmdopts, needs, sh, task
@@ -89,7 +89,7 @@ def wait_for_server(server, port):
while attempts < 120:
try:
connection = six.moves.http_client.HTTPConnection(server, port, timeout=10)
connection = HTTPConnection(server, port, timeout=10)
connection.request('GET', '/')
response = connection.getresponse()
@@ -115,7 +115,7 @@ def wait_for_test_servers():
if not ready:
msg = colorize(
"red",
"Could not contact {} test server".format(service)
f"Could not contact {service} test server"
)
print(msg)
sys.exit(1)
@@ -127,7 +127,7 @@ def is_mongo_running():
"""
# The mongo command will connect to the service,
# failing with a non-zero exit code if it cannot connect.
output = os.popen('mongo --host {} --eval "print(\'running\')"'.format(Env.MONGO_HOST)).read()
output = os.popen(f'mongo --host {Env.MONGO_HOST} --eval "print(\'running\')"').read()
return output and "running" in output

26
pavelib/utils/test/suites/bokchoy_suite.py
View File

@@ -68,7 +68,7 @@ def load_courses(options):
`test_root/courses/`.
"""
if 'imports_dir' in options:
msg = colorize('green', "Importing courses from {}...".format(options.imports_dir))
msg = colorize('green', f"Importing courses from {options.imports_dir}...")
print(msg)
sh(
@@ -239,7 +239,7 @@ class BokChoyTestSuite(TestSuite):
# Clean up data we created in the databases
msg = colorize('green', "Cleaning up databases...")
print(msg)
sh("./manage.py lms --settings {settings} flush --traceback --noinput".format(settings=Env.SETTINGS))
sh(f"./manage.py lms --settings {Env.SETTINGS} flush --traceback --noinput")
clear_mongo()
@property
@@ -247,12 +247,12 @@ class BokChoyTestSuite(TestSuite):
"""
Construct the proper combination of multiprocessing, XUnit XML file, color, and verbosity for use with pytest.
"""
command = ["--junitxml={}".format(self.xunit_report)]
command = [f"--junitxml={self.xunit_report}"]
if self.num_processes != 1:
# Construct "multiprocess" pytest command
command += [
"-n {}".format(self.num_processes),
f"-n {self.num_processes}",
"--color=no",
]
if self.verbosity < 1:
@@ -260,7 +260,7 @@ class BokChoyTestSuite(TestSuite):
elif self.verbosity > 1:
command.append("--verbose")
if self.eval_attr:
command.append("-a '{}'".format(self.eval_attr))
command.append(f"-a '{self.eval_attr}'")
return command
@@ -298,13 +298,13 @@ class BokChoyTestSuite(TestSuite):
# Construct the pytest command, specifying where to save
# screenshots and XUnit XML reports
cmd = [
"DEFAULT_STORE={}".format(self.default_store),
"SAVED_SOURCE_DIR='{}'".format(self.log_dir),
"SCREENSHOT_DIR='{}'".format(self.log_dir),
"BOK_CHOY_HAR_DIR='{}'".format(self.har_dir),
"BOKCHOY_A11Y_CUSTOM_RULES_FILE='{}'".format(self.a11y_file),
"SELENIUM_DRIVER_LOG_DIR='{}'".format(self.log_dir),
"VERIFY_XSS='{}'".format(self.verify_xss),
f"DEFAULT_STORE={self.default_store}",
f"SAVED_SOURCE_DIR='{self.log_dir}'",
f"SCREENSHOT_DIR='{self.log_dir}'",
f"BOK_CHOY_HAR_DIR='{self.har_dir}'",
f"BOKCHOY_A11Y_CUSTOM_RULES_FILE='{self.a11y_file}'",
f"SELENIUM_DRIVER_LOG_DIR='{self.log_dir}'",
f"VERIFY_XSS='{self.verify_xss}'",
]
if self.save_screenshots:
cmd.append("NEEDLE_SAVE_BASELINE=True")
@@ -313,7 +313,7 @@ class BokChoyTestSuite(TestSuite):
"coverage",
"run",
]
cmd.append("--rcfile={}".format(self.coveragerc))
cmd.append(f"--rcfile={self.coveragerc}")
else:
cmd += [
"python",

12
pavelib/utils/test/suites/js_suite.py
View File

@@ -68,8 +68,8 @@ class JsTestSubSuite(TestSuite):
except ValueError:
self.test_conf_file = Env.KARMA_CONFIG_FILES[0]
self.coverage_report = self.report_dir / 'coverage-{suite}.xml'.format(suite=self.test_id)
self.xunit_report = self.report_dir / 'javascript_xunit-{suite}.xml'.format(suite=self.test_id)
self.coverage_report = self.report_dir / f'coverage-{self.test_id}.xml'
self.xunit_report = self.report_dir / f'javascript_xunit-{self.test_id}.xml'
@property
def cmd(self):
@@ -84,17 +84,17 @@ class JsTestSubSuite(TestSuite):
self.test_conf_file,
"--single-run={}".format('false' if self.mode == 'dev' else 'true'),
"--capture-timeout=60000",
"--junitreportpath={}".format(self.xunit_report),
"--browsers={}".format(Env.KARMA_BROWSER),
f"--junitreportpath={self.xunit_report}",
f"--browsers={Env.KARMA_BROWSER}",
]
if self.port:
cmd.append("--port={}".format(self.port))
cmd.append(f"--port={self.port}")
if self.run_under_coverage:
cmd.extend([
"--coverage",
"--coveragereportpath={}".format(self.coverage_report),
f"--coveragereportpath={self.coverage_report}",
])
return cmd

30
pavelib/utils/test/suites/pytest_suite.py
View File

@@ -108,9 +108,9 @@ class PytestSuite(TestSuite):
if self.with_wtw:
opts.extend([
'--wtw',
'{}/{}'.format(COVERAGE_CACHE_BASEPATH, WHO_TESTS_WHAT_DIFF),
f'{COVERAGE_CACHE_BASEPATH}/{WHO_TESTS_WHAT_DIFF}',
'--wtwdb',
'{}/{}'.format(COVERAGE_CACHE_BASEPATH, COVERAGE_CACHE_BASELINE)
f'{COVERAGE_CACHE_BASEPATH}/{COVERAGE_CACHE_BASELINE}'
])
return opts
@@ -160,8 +160,8 @@ class SystemTestSuite(PytestSuite):
'-Wd',
'-m',
'pytest',
'--ds={}'.format('{}.envs.{}'.format(self.root, self.settings)),
"--junitxml={}".format(self.xunit_report),
'--ds={}'.format(f'{self.root}.envs.{self.settings}'),
f"--junitxml={self.xunit_report}",
])
cmd.extend(self.test_options_flags)
if self.verbosity < 1:
@@ -186,7 +186,7 @@ class SystemTestSuite(PytestSuite):
for ip in self.xdist_ip_addresses.split(','):
# Propogate necessary env vars to xdist containers
env_var_cmd = 'export DJANGO_SETTINGS_MODULE={} DISABLE_COURSEENROLLMENT_HISTORY={} PYTHONHASHSEED=0'\
.format('{}.envs.{}'.format(self.root, self.settings),
.format(f'{self.root}.envs.{self.settings}',
self.disable_courseenrollment_history)
xdist_string = '--tx {}*ssh="jenkins@{} -o StrictHostKeyChecking=no"' \
'//python="source edx-venv-{}/edx-venv/bin/activate; {}; python"' \
@@ -194,19 +194,19 @@ class SystemTestSuite(PytestSuite):
.format(xdist_remote_processes, ip, Env.PYTHON_VERSION, env_var_cmd)
cmd.append(xdist_string)
for rsync_dir in Env.rsync_dirs():
cmd.append('--rsyncdir {}'.format(rsync_dir))
cmd.append(f'--rsyncdir {rsync_dir}')
else:
if self.processes == -1:
cmd.append('-n auto')
cmd.append('--dist=loadscope')
elif self.processes != 0:
cmd.append('-n {}'.format(self.processes))
cmd.append(f'-n {self.processes}')
cmd.append('--dist=loadscope')
if not self.randomize:
cmd.append('-p no:randomly')
if self.eval_attr:
cmd.append("-a '{}'".format(self.eval_attr))
cmd.append(f"-a '{self.eval_attr}'")
cmd.extend(self.passthrough_options)
cmd.append(self.test_id)
@@ -226,17 +226,17 @@ class SystemTestSuite(PytestSuite):
# thereby making sure that we load any django models that are
# only defined in test files.
default_test_globs = [
"{system}/djangoapps/*".format(system=self.root),
f"{self.root}/djangoapps/*",
"common/djangoapps/*",
"openedx/core/djangoapps/*",
"openedx/tests/*",
"openedx/core/lib/*",
]
if self.root in ('lms', 'cms'):
default_test_globs.append("{system}/lib/*".format(system=self.root))
default_test_globs.append(f"{self.root}/lib/*")
if self.root == 'lms':
default_test_globs.append("{system}/tests.py".format(system=self.root))
default_test_globs.append(f"{self.root}/tests.py")
default_test_globs.append("openedx/core/djangolib/*")
default_test_globs.append("openedx/core/tests/*")
default_test_globs.append("openedx/features")
@@ -288,7 +288,7 @@ class LibTestSuite(PytestSuite):
'-Wd',
'-m',
'pytest',
'--junitxml={}'.format(self.xunit_report),
f'--junitxml={self.xunit_report}',
])
cmd.extend(self.passthrough_options + self.test_options_flags)
if self.verbosity < 1:
@@ -322,7 +322,7 @@ class LibTestSuite(PytestSuite):
.format(xdist_remote_processes, ip, Env.PYTHON_VERSION, env_var_cmd)
cmd.append(xdist_string)
for rsync_dir in Env.rsync_dirs():
cmd.append('--rsyncdir {}'.format(rsync_dir))
cmd.append(f'--rsyncdir {rsync_dir}')
# "--rsyncdir" throws off the configuration root, set it explicitly
if 'common/lib' in self.test_id:
cmd.append('--rootdir=common/lib')
@@ -334,13 +334,13 @@ class LibTestSuite(PytestSuite):
cmd.append('-n auto')
cmd.append('--dist=loadscope')
elif self.processes != 0:
cmd.append('-n {}'.format(self.processes))
cmd.append(f'-n {self.processes}')
cmd.append('--dist=loadscope')
if not self.randomize:
cmd.append("-p no:randomly")
if self.eval_attr:
cmd.append("-a '{}'".format(self.eval_attr))
cmd.append(f"-a '{self.eval_attr}'")
cmd.append(self.test_id)

4
pavelib/utils/test/suites/suite.py
View File

@@ -41,7 +41,7 @@ class TestSuite:
i.e. Checking for and defining required directories.
"""
print("\nSetting up for {suite_name}".format(suite_name=self.root))
print(f"\nSetting up for {self.root}")
self.failed_suites = []
def __exit__(self, exc_type, exc_value, traceback):
@@ -54,7 +54,7 @@ class TestSuite:
i.e. Cleaning mongo after the lms tests run.
"""
print("\nCleaning up after {suite_name}".format(suite_name=self.root))
print(f"\nCleaning up after {self.root}")
@property
def cmd(self):

2
pavelib/utils/test/utils.py
View File

@@ -63,7 +63,7 @@ def clean_dir(directory):
"""
# We delete the files but preserve the directory structure
# so that coverage.py has a place to put the reports.
sh('find {dir} -type f -delete'.format(dir=directory))
sh(f'find {directory} -type f -delete')
@task

2
pavelib/utils/timer.py
View File

@@ -55,7 +55,7 @@ def timed(wrapped, instance, args, kwargs): # pylint: disable=unused-argument
log_message = {
'python_version': sys.version,
'task': "{}.{}".format(wrapped.__module__, wrapped.__name__),
'task': f"{wrapped.__module__}.{wrapped.__name__}",
'args': [repr(arg) for arg in args],
'kwargs': {key: repr(value) for key, value in kwargs.items()},
'started_at': start.isoformat(' '),

13
scripts/xblock/xblock_counts.py
View File

@@ -1,5 +1,3 @@
import argparse
import csv
import json
@@ -8,7 +6,6 @@ import sys
from datetime import datetime
import requests
from six import text_type
# Keys for the CSV and JSON interpretation
PAGINATION_KEY = 'pagination'
@@ -163,14 +160,14 @@ def _get_block_types_from_json_file(xblock_json_file):
if not os.path.isfile(xblock_json_file):
print('xBlock configuration file does not exist: %s' % xblock_json_file)
sys.exit(2)
with open(xblock_json_file, 'r') as json_file:
with open(xblock_json_file) as json_file:
type_set = set()
try:
json_data = json.loads(json_file.read())
except ValueError as e:
print('xBlock configuration file does not match the expected layout and is '
'missing "data" list: %s' % xblock_json_file)
sys.exit(text_type(e))
sys.exit(str(e))
if 'data' in json_data:
xblock_type_list = json_data['data']
for xblock in xblock_type_list:
@@ -213,11 +210,11 @@ def _get_course_block_counts(auth_token, block_url):
Returns:
dict: A dictionary containing the Block counts
"""
headers = {'Authorization': 'Bearer {}'.format(auth_token)}
headers = {'Authorization': f'Bearer {auth_token}'}
response = requests.get(block_url, headers=headers)
if response.status_code != 200:
print("url {} returned status code {}".format(block_url, response.status_code))
print(f"url {block_url} returned status code {response.status_code}")
return {}
response_json = response.json()
@@ -386,4 +383,4 @@ if __name__ == "__main__":
if len(course_data) > 0:
write_block_summary_report(course_data)
write_course_block_detail_report(course_data)
print('Start time: %s Total run time: %s' % (str(start_time), str(datetime.now() - start_time)))
print('Start time: {} Total run time: {}'.format(str(start_time), str(datetime.now() - start_time)))

12
scripts/xdist/find_order_dependent_test_failures.py
View File

@@ -78,7 +78,7 @@ def main(log_file, test_suite, fast, verbose):
test_list_with_failures, pytest_command = _find_fewest_tests_with_failures(failing_test_list, 'ALL')
if test_list_with_failures:
print('Found failures running {} tests.'.format(len(test_list_with_failures)))
print('Use: {}'.format(pytest_command))
print(f'Use: {pytest_command}')
return
if fast_option:
@@ -102,9 +102,9 @@ def _strip_console_for_tests_with_failure(log_file, test_suite):
worker_test_dict = {}
test_base_included = {}
failing_worker_num = None
with io.open(log_file, 'r') as console_file:
with open(log_file, 'r') as console_file:
for line in console_file:
regex_search = re.search(r'\[gw(\d+)] (PASSED|FAILED|SKIPPED|ERROR) (\S+)'.format(test_suite), line)
regex_search = re.search(fr'\[gw(\d+)] (PASSED|FAILED|SKIPPED|ERROR) (\S+)', line)
if regex_search:
worker_num_string = regex_search.group(1)
pass_fail_string = regex_search.group(2)
@@ -113,7 +113,7 @@ def _strip_console_for_tests_with_failure(log_file, test_suite):
test = regex_search.group(3)
if test_suite == "commonlib-unit":
if "pavelib" not in test and not test.startswith('scripts'):
test = u"common/lib/{}".format(test)
test = f"common/lib/{test}"
if fast_option and pass_fail_string == 'PASSED':
# fast option will only take one test per class or module, in case
# the failure is a setup/teardown failure.
@@ -134,7 +134,7 @@ def _get_pytest_command(output_file_name):
"""
Return the pytest command to run.
"""
return "pytest -p 'no:randomly' `cat {}`".format(output_file_name)
return f"pytest -p 'no:randomly' `cat {output_file_name}`"
def _run_tests_and_check_for_failures(output_file_name):
@@ -169,7 +169,7 @@ def _create_and_check_test_files_for_failures(test_list, test_type):
# to the command line, but this keeps the verbose output cleaner.
temp_file = tempfile.NamedTemporaryFile(prefix=output_file_name, dir=OUTPUT_FOLDER_NAME, delete=False)
with io.open(temp_file.name, 'w') as output_file:
with open(temp_file.name, 'w') as output_file:
for line in test_list:
output_file.write(line + "\n")
temp_file.close()

10
scripts/xdist/get_worker_test_list.py
View File

@@ -30,9 +30,9 @@ import click
)
def main(log_file, test_suite):
worker_test_dict = {}
with io.open(log_file, 'r') as console_file:
with open(log_file, 'r') as console_file:
for line in console_file:
regex_search = re.search(r'\[gw(\d+)] (PASSED|FAILED|SKIPPED|ERROR) (\S+)'.format(test_suite), line)
regex_search = re.search(fr'\[gw(\d+)] (PASSED|FAILED|SKIPPED|ERROR) (\S+)', line)
if regex_search:
worker_num_string = regex_search.group(1)
if worker_num_string not in worker_test_dict:
@@ -40,7 +40,7 @@ def main(log_file, test_suite):
test = regex_search.group(3)
if test_suite == "commonlib-unit":
if "pavelib" not in test and not test.startswith('scripts'):
test = u"common/lib/{}".format(test)
test = f"common/lib/{test}"
worker_test_dict[worker_num_string].append(test)
output_folder_name = "worker_list_files"
@@ -49,8 +49,8 @@ def main(log_file, test_suite):
os.mkdir(output_folder_name)
for worker_num in worker_test_dict:
output_file_name = "{}/{}_gw{}_test_list.txt".format(output_folder_name, test_suite, worker_num)
with io.open(output_file_name, 'w') as output_file:
output_file_name = f"{output_folder_name}/{test_suite}_gw{worker_num}_test_list.txt"
with open(output_file_name, 'w') as output_file:
for line in worker_test_dict[worker_num]:
output_file.write(line + "\n")

15
scripts/xdist/pytest_worker_manager.py
View File

@@ -12,7 +12,6 @@ from botocore.config import Config
from botocore.exceptions import ClientError
import socket
from multiprocessing import Pool
from six.moves import range
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@@ -41,7 +40,7 @@ class PytestWorkerManager():
Spins up workers and generates two .txt files, containing the IP/ arns
of the new workers.
"""
logging.info("Spinning up {} workers".format(number_of_workers))
logging.info(f"Spinning up {number_of_workers} workers")
worker_instance_ids = []
for retry in range(1, self.MAX_RUN_WORKER_RETRIES + 1):
@@ -71,11 +70,11 @@ class PytestWorkerManager():
# Handle AWS throttling with an exponential backoff
if retry == self.MAX_RUN_WORKER_RETRIES:
raise Exception(
"MAX_RUN_WORKER_RETRIES ({}) reached while spinning up workers due to AWS throttling.".format(self.MAX_RUN_WORKER_RETRIES)
f"MAX_RUN_WORKER_RETRIES ({self.MAX_RUN_WORKER_RETRIES}) reached while spinning up workers due to AWS throttling."
)
logger.info("Hit error: {}. Retrying".format(err))
logger.info(f"Hit error: {err}. Retrying")
countdown = 2 ** retry
logger.info("Sleeping for {} seconds".format(countdown))
logger.info(f"Sleeping for {countdown} seconds")
time.sleep(countdown)
else:
break
@@ -113,7 +112,7 @@ class PytestWorkerManager():
raise Exception(
"Timed out waiting to spin up all workers."
)
logger.info("Successfully booted up {} workers.".format(number_of_workers))
logger.info(f"Successfully booted up {number_of_workers} workers.")
not_ready_ip_addresses = ip_addresses[:]
logger.info("Checking ssh connection to workers.")
@@ -140,13 +139,13 @@ class PytestWorkerManager():
# Generate .txt files containing IP addresses and instance ids
ip_list_string = ",".join(ip_addresses)
logger.info("Worker IP list: {}".format(ip_list_string))
logger.info(f"Worker IP list: {ip_list_string}")
ip_list_file = open("pytest_worker_ips.txt", "w")
ip_list_file.write(ip_list_string)
ip_list_file.close()
worker_instance_id_list_string = ",".join(worker_instance_ids)
logger.info("Worker Instance Id list: {}".format(worker_instance_id_list_string))
logger.info(f"Worker Instance Id list: {worker_instance_id_list_string}")
worker_arn_file = open("pytest_worker_instance_ids.txt", "w")
worker_arn_file.write(worker_instance_id_list_string)
worker_arn_file.close()

1
scripts/xsslint/tests/templates/test.py
View File

@@ -1,5 +1,4 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Testing encoding on second line does not cause violation
message = "<script>alert('XSS');</script>"
x = "<string>{}</strong>".format(message)

4
scripts/xsslint/tests/test_linters.py
View File

@@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
"""
Tests for linters.py
"""
@@ -8,7 +7,6 @@ import textwrap
from unittest import TestCase
from ddt import data, ddt
from six.moves import range, zip
from xsslint.linters import (
JavaScriptLinter, MakoTemplateLinter,
@@ -69,7 +67,7 @@ class TestLinter(TestCase):
# Print violations if the lengths are different.
if len(results.violations) != len(rules):
for violation in results.violations:
print("Found violation: {}".format(violation.rule))
print(f"Found violation: {violation.rule}")
assert len(results.violations) == len(rules)
for violation, rule in zip(results.violations, rules):

21
scripts/xsslint/tests/test_main.py
View File

@@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
"""
Tests for main.py
"""
@@ -6,10 +5,10 @@ Tests for main.py
import json
import re
from six import StringIO
from io import StringIO
from unittest import TestCase
import mock
from unittest import mock
from xsslint.linters import JavaScriptLinter, MakoTemplateLinter, PythonLinter, UnderscoreTemplateLinter
from xsslint.main import _build_ruleset, _lint
@@ -79,9 +78,9 @@ class TestXSSLinter(TestCase):
output = self.out.getvalue()
# Assert violation details are displayed.
assert re.search('test\\.html.*{}'.format(self.ruleset.mako_missing_default.rule_id), output) is not None
assert re.search('test\\.js.*{}'.format(self.ruleset.javascript_concat_html.rule_id), output) is not None
assert re.search('test\\.js.*{}'.format(self.ruleset.underscore_not_escaped.rule_id), output) is not None
assert re.search(f'test\\.html.*{self.ruleset.mako_missing_default.rule_id}', output) is not None
assert re.search(f'test\\.js.*{self.ruleset.javascript_concat_html.rule_id}', output) is not None
assert re.search(f'test\\.js.*{self.ruleset.underscore_not_escaped.rule_id}', output) is not None
lines_with_rule = 0
lines_without_rule = 0 # Output with verbose setting only.
for underscore_match in re.finditer(r'test\.underscore:.*\n', output):
@@ -91,8 +90,8 @@ class TestXSSLinter(TestCase):
lines_without_rule += 1
assert lines_with_rule >= 1
assert lines_without_rule == 0
assert re.search('test\\.py.*{}'.format(self.ruleset.python_parse_error.rule_id), output) is None
assert re.search('test\\.py.*{}'.format(self.ruleset.python_wrap_html.rule_id), output) is not None
assert re.search(f'test\\.py.*{self.ruleset.python_parse_error.rule_id}', output) is None
assert re.search(f'test\\.py.*{self.ruleset.python_wrap_html.rule_id}', output) is not None
# Assert no rule totals.
assert re.search('{}:\\s*{} violations'.format(self.ruleset.python_parse_error.rule_id, 0), output) is None
# Assert final total
@@ -150,7 +149,7 @@ class TestXSSLinter(TestCase):
)
output = self.out.getvalue()
assert re.search('test\\.py.*{}'.format(self.ruleset.python_wrap_html.rule_id), output) is not None
assert re.search(f'test\\.py.*{self.ruleset.python_wrap_html.rule_id}', output) is not None
# Assert totals output.
assert re.search('{}:\\s*{} violations'.format(self.ruleset.python_parse_error.rule_id, 0), output) is not None
@@ -176,7 +175,7 @@ class TestXSSLinter(TestCase):
)
output = self.out.getvalue()
assert re.search('test\\.py.*{}'.format(self.ruleset.python_wrap_html.rule_id), output) is not None
assert re.search(f'test\\.py.*{self.ruleset.python_wrap_html.rule_id}', output) is not None
# Find something that looks like pretty-printed JSON
json_match = re.search(r'\n\{.*\n\}', output, re.DOTALL)
@@ -207,7 +206,7 @@ class TestXSSLinter(TestCase):
output = self.out.getvalue()
# Assert file with rule is not output.
assert re.search('test\\.py.*{}'.format(self.ruleset.python_wrap_html.rule_id), output) is None
assert re.search(f'test\\.py.*{self.ruleset.python_wrap_html.rule_id}', output) is None
# Assert file is output.
assert re.search('test\\.py', output) is not None

2
scripts/xsslint/tests/test_utils.py
View File

@@ -1,5 +1,3 @@
from unittest import TestCase
from ddt import data, ddt

6
scripts/xsslint/xsslint/django_linter.py
View File

@@ -12,7 +12,7 @@ class TransExpression(Expression):
"""
def __init__(self, ruleset, results, *args, **kwargs):
super(TransExpression, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.string_lines = StringLines(kwargs['template'])
self.ruleset = ruleset
self.results = results
@@ -202,7 +202,7 @@ class BlockTransExpression(Expression):
The expression handling blocktrans tag
"""
def __init__(self, ruleset, results, *args, **kwargs):
super(BlockTransExpression, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.string_lines = StringLines(kwargs['template'])
self.ruleset = ruleset
self.results = results
@@ -319,7 +319,7 @@ class HtmlInterpolateExpression(Expression):
The expression handling interplate_html tag
"""
def __init__(self, ruleset, results, *args, **kwargs):
super(HtmlInterpolateExpression, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.string_lines = StringLines(kwargs['template'])
self.ruleset = ruleset
self.results = results

18
scripts/xsslint/xsslint/linters.py
View File

@@ -16,7 +16,7 @@ from xsslint.utils import Expression, ParseString, StringLines, is_skip_dir
from xsslint.django_linter import TransExpression, BlockTransExpression, HtmlInterpolateExpression
class BaseLinter(object):
class BaseLinter:
"""
BaseLinter provides some helper functions that are used by multiple linters.
@@ -53,7 +53,7 @@ class BaseLinter(object):
A string containing the files contents.
"""
with io.open(file_full_path, 'r') as input_file:
with open(file_full_path, 'r') as input_file:
file_contents = input_file.read()
return file_contents
@@ -177,7 +177,7 @@ class BaseLinter(object):
"""
if self.LINE_COMMENT_DELIM is not None:
line_start_index = StringLines(template).index_to_line_start_index(start_index)
uncommented_line_start_index_regex = re.compile(r"^(?!\s*{})".format(self.LINE_COMMENT_DELIM), re.MULTILINE)
uncommented_line_start_index_regex = re.compile(fr"^(?!\s*{self.LINE_COMMENT_DELIM})", re.MULTILINE)
# Finds the line start index of the first uncommented line, including the current line.
match = uncommented_line_start_index_regex.search(template, line_start_index)
if match is None:
@@ -207,7 +207,7 @@ class UnderscoreTemplateLinter(BaseLinter):
"""
Init method.
"""
super(UnderscoreTemplateLinter, self).__init__()
super().__init__()
self._skip_underscore_dirs = skip_dirs or ()
def process_file(self, directory, file_name):
@@ -335,7 +335,7 @@ class JavaScriptLinter(BaseLinter):
"""
Init method.
"""
super(JavaScriptLinter, self).__init__()
super().__init__()
self.underscore_linter = underscore_linter
self.ruleset = self.ruleset + self.underscore_linter.ruleset
self._skip_javascript_dirs = javascript_skip_dirs or ()
@@ -468,7 +468,7 @@ class JavaScriptLinter(BaseLinter):
"""
# Ignores calls starting with "HtmlUtils.", because those are safe
regex = re.compile(r"(?<!HtmlUtils).(?:{})\(".format(function_names))
regex = re.compile(fr"(?<!HtmlUtils).(?:{function_names})\(")
for function_match in regex.finditer(file_contents):
is_violation = True
expression = self._get_expression_for_function(file_contents, function_match)
@@ -709,7 +709,7 @@ class PythonLinter(BaseLinter):
"""
Init method.
"""
super(PythonLinter, self).__init__()
super().__init__()
self._skip_python_dirs = skip_dirs or ()
def process_file(self, directory, file_name):
@@ -887,7 +887,7 @@ class MakoTemplateLinter(BaseLinter):
"""
Init method.
"""
super(MakoTemplateLinter, self).__init__()
super().__init__()
self.javascript_linter = javascript_linter
self.python_linter = python_linter
self.ruleset = self.ruleset + self.javascript_linter.ruleset + self.python_linter.ruleset
@@ -1490,7 +1490,7 @@ class DjangoTemplateLinter(BaseLinter):
"""
Init method.
"""
super(DjangoTemplateLinter, self).__init__()
super().__init__()
self._skip_django_dirs = skip_dirs or ()
def process_file(self, directory, file_name):

4
scripts/xsslint/xsslint/main.py
View File

@@ -120,7 +120,7 @@ def _lint(file_or_dir, template_linters, options, summary_results, out):
if os.path.exists(file_or_dir):
directory = file_or_dir
else:
raise ValueError("Path [{}] is not a valid file or directory.".format(file_or_dir))
raise ValueError(f"Path [{file_or_dir}] is not a valid file or directory.")
_process_os_dirs(directory, template_linters, options, summary_results, out)
summary_results.print_results(options, out)
@@ -177,7 +177,7 @@ def main():
}
template_linters = getattr(config, 'LINTERS', ())
if not template_linters:
raise ValueError("LINTERS is empty or undefined in the config module ({}).".format(args.config))
raise ValueError(f"LINTERS is empty or undefined in the config module ({args.config}).")
ruleset = _build_ruleset(template_linters)
summary_results = SummaryResults(ruleset)

13
scripts/xsslint/xsslint/reporting.py
View File

@@ -7,12 +7,11 @@ import json
import os
import re
from six.moves import range
from xsslint.utils import StringLines
class RuleViolation(object):
class RuleViolation:
"""
Base class representing a rule violation which can be used for reporting.
"""
@@ -97,7 +96,7 @@ class RuleViolation(object):
_options: ignored
out: output file
"""
print("{}: {}".format(self.full_path, self.rule.rule_id), file=out)
print(f"{self.full_path}: {self.rule.rule_id}", file=out)
class ExpressionRuleViolation(RuleViolation):
@@ -117,7 +116,7 @@ class ExpressionRuleViolation(RuleViolation):
expression: The Expression that was in violation.
"""
super(ExpressionRuleViolation, self).__init__(rule)
super().__init__(rule)
self.expression = expression
self.start_line = 0
self.start_column = 0
@@ -235,7 +234,7 @@ class ExpressionRuleViolation(RuleViolation):
), file=out)
class SummaryResults(object):
class SummaryResults:
"""
Contains the summary results for all violations.
"""
@@ -295,7 +294,7 @@ class SummaryResults(object):
# matches output of eslint for simplicity
print("", file=out)
print("{} violations total".format(self.total_violations), file=out)
print(f"{self.total_violations} violations total", file=out)
def _print_json_format(self, options, out):
"""
@@ -315,7 +314,7 @@ class SummaryResults(object):
)
class FileResults(object):
class FileResults:
"""
Contains the results, or violations, for a file.
"""

4
scripts/xsslint/xsslint/rules.py
View File

@@ -1,4 +1,4 @@
class Rule(object):
class Rule:
def __init__(self, rule_id):
self.rule_id = rule_id
@@ -6,7 +6,7 @@ class Rule(object):
return self.rule_id == other.rule_id
class RuleSet(object):
class RuleSet:
def __init__(self, **kwargs):
self.rules = {}
for k, v in kwargs.items():

6
scripts/xsslint/xsslint/utils.py
View File

@@ -26,7 +26,7 @@ def is_skip_dir(skip_dirs, directory):
return False
class StringLines(object):
class StringLines:
"""
StringLines provides utility methods to work with a string in terms of
lines. As an example, it can convert an index into a line number or column
@@ -199,7 +199,7 @@ class StringLines(object):
return len(self._line_start_indexes)
class ParseString(object):
class ParseString:
"""
ParseString is the result of parsing a string out of a template.
@@ -303,7 +303,7 @@ class ParseString(object):
}
class Expression(object):
class Expression:
"""
Represents an arbitrary expression.

10
scripts/xsslint/xsslint/visitors.py
View File

@@ -35,7 +35,7 @@ class BaseVisitor(ast.NodeVisitor):
results: A file results objects to which violations will be added.
"""
super(BaseVisitor, self).__init__()
super().__init__()
self.file_contents = file_contents
self.lines = StringLines(self.file_contents)
self.results = results
@@ -98,7 +98,7 @@ class HtmlStringVisitor(BaseVisitor):
skip_wrapped_html: True if visitor should skip strings wrapped with
HTML() or Text(), and False otherwise.
"""
super(HtmlStringVisitor, self).__init__(file_contents, results)
super().__init__(file_contents, results)
self.skip_wrapped_html = skip_wrapped_html
self.unsafe_html_string_nodes = []
self.over_escaped_entity_string_nodes = []
@@ -150,7 +150,7 @@ class ContainsFormatVisitor(BaseVisitor):
results: A file results objects to which violations will be added.
"""
super(ContainsFormatVisitor, self).__init__(file_contents, results)
super().__init__(file_contents, results)
self.contains_format_call = False
def visit_Attribute(self, node):
@@ -185,7 +185,7 @@ class FormatInterpolateVisitor(BaseVisitor):
results: A file results objects to which violations will be added.
"""
super(FormatInterpolateVisitor, self).__init__(file_contents, results)
super().__init__(file_contents, results)
self.interpolates_text_or_html = False
self.format_caller_node = None
@@ -223,7 +223,7 @@ class FormatInterpolateVisitor(BaseVisitor):
"""
if self.interpolates_text_or_html is False:
if self.format_caller_node is not node:
super(FormatInterpolateVisitor, self).generic_visit(node)
super().generic_visit(node)
class OuterFormatVisitor(BaseVisitor):