Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch Jenkinsfile over to xdist

Browse Source
This commit is contained in:
Michael Youngstrom
2018-07-25 16:05:47 -04:00
committed by Michael Youngstrom
parent 9ad0f2b838
commit b4fcbbd363
9 changed files with 304 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files

326
Jenkinsfile vendored
View File

@@ -1,42 +1,46 @@
def runPythonTests() {
ansiColor('gnome-terminal') {
sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) {
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${sha1}']],
doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [],
userRemoteConfigs: [[credentialsId: 'jenkins-worker',
refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*',
url: 'git@github.com:edx/edx-platform.git']]]
console_output = sh(returnStdout: true, script: 'bash scripts/all-tests.sh').trim()
dir('stdout') {
writeFile file: "${TEST_SUITE}-${SHARD}-stdout.log", text: console_output
}
stash includes: 'reports/**/*coverage*', name: "${TEST_SUITE}-${SHARD}-reports"
sshagent(credentials: ['jenkins-worker', 'jenkins-worker-pem'], ignoreMissing: true) {
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${ghprbActualCommit}']],
doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [],
userRemoteConfigs: [[credentialsId: 'jenkins-worker',
refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*',
url: 'git@github.com:edx/edx-platform.git']]]
console_output = sh(returnStdout: true, script: 'bash scripts/all-tests.sh').trim()
dir('stdout') {
writeFile file: "${TEST_SUITE}-stdout.log", text: console_output
}
stash includes: 'reports/**/*coverage*', name: "${TEST_SUITE}-reports"
}
}
def savePythonTestArtifacts() {
def pythonTestCleanup() {
archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/**/*,test_root/log/**/*.log,**/nosetests.xml,stdout/*.log,*.log'
junit '**/nosetests.xml'
sh '''source $HOME/edx-venv/bin/activate
bash scripts/xdist/terminate_xdist_nodes.sh'''
}
pipeline {
agent { label "coverage-worker" }
options {
timestamps()
timeout(75)
timeout(60)
}
environment {
XDIST_CONTAINER_SUBNET = credentials('XDIST_CONTAINER_SUBNET')
XDIST_CONTAINER_SECURITY_GROUP = credentials('XDIST_CONTAINER_SECURITY_GROUP')
XDIST_CONTAINER_TASK_NAME = "jenkins-worker-task"
XDIST_GIT_BRANCH = "${ghprbActualCommit}"
}
stages {
stage('Run Tests') {
parallel {
stage('lms-unit-1') {
stage("lms-unit") {
agent { label "jenkins-worker" }
environment {
SHARD = 1
TEST_SUITE = 'lms-unit'
TEST_SUITE = "lms-unit"
XDIST_NUM_TASKS = 10
XDIST_REMOTE_NUM_PROCESSES = 2
}
steps {
script {
@@ -46,35 +50,17 @@ pipeline {
post {
always {
script {
savePythonTestArtifacts()
pythonTestCleanup()
}
}
}
}
stage('lms-unit-2') {
stage("cms-unit") {
agent { label "jenkins-worker" }
environment {
SHARD = 2
TEST_SUITE = 'lms-unit'
}
steps{
script {
runPythonTests()
}
}
post {
always {
script {
savePythonTestArtifacts()
}
}
}
}
stage('lms-unit-3') {
agent { label "jenkins-worker" }
environment {
SHARD = 3
TEST_SUITE = 'lms-unit'
TEST_SUITE = "cms-unit"
XDIST_NUM_TASKS = 4
XDIST_REMOTE_NUM_PROCESSES = 2
}
steps {
script {
@@ -84,16 +70,17 @@ pipeline {
post {
always {
script {
savePythonTestArtifacts()
pythonTestCleanup()
}
}
}
}
stage('lms-unit-4') {
stage("commonlib-unit") {
agent { label "jenkins-worker" }
environment {
SHARD = 4
TEST_SUITE = 'lms-unit'
TEST_SUITE = "commonlib-unit"
XDIST_NUM_TASKS = 3
XDIST_REMOTE_NUM_PROCESSES = 2
}
steps {
script {
@@ -103,216 +90,7 @@ pipeline {
post {
always {
script {
savePythonTestArtifacts()
}
}
}
}
stage('lms-unit-5') {
agent { label "jenkins-worker" }
environment {
SHARD = 5
TEST_SUITE = 'lms-unit'
}
steps {
script {
runPythonTests()
}
}
post {
always {
script {
savePythonTestArtifacts()
}
}
}
}
stage('lms-unit-6') {
agent { label "jenkins-worker" }
environment {
SHARD = 6
TEST_SUITE = 'lms-unit'
}
steps {
script {
runPythonTests()
}
}
post {
always {
script {
savePythonTestArtifacts()
}
}
}
}
stage('lms-unit-7') {
agent { label "jenkins-worker" }
environment {
SHARD = 7
TEST_SUITE = 'lms-unit'
}
steps {
script {
runPythonTests()
}
}
post {
always {
script {
savePythonTestArtifacts()
}
}
}
}
stage('lms-unit-8') {
agent { label "jenkins-worker" }
environment {
SHARD = 8
TEST_SUITE = 'lms-unit'
}
steps {
script {
runPythonTests()
}
}
post {
always {
script {
savePythonTestArtifacts()
}
}
}
}
stage('lms-unit-9') {
agent { label "jenkins-worker" }
environment {
SHARD = 9
TEST_SUITE = 'lms-unit'
}
steps {
script {
runPythonTests()
}
}
post {
always {
script {
savePythonTestArtifacts()
}
}
}
}
stage('lms-unit-10') {
agent { label "jenkins-worker" }
environment {
SHARD = 10
TEST_SUITE = 'lms-unit'
}
steps {
script {
runPythonTests()
}
}
post {
always {
script {
savePythonTestArtifacts()
}
}
}
}
stage('cms-unit-1') {
agent { label "jenkins-worker" }
environment {
SHARD = 1
TEST_SUITE = 'cms-unit'
}
steps {
script {
runPythonTests()
}
}
post {
always {
script {
savePythonTestArtifacts()
}
}
}
}
stage('cms-unit-2') {
agent { label "jenkins-worker" }
environment {
SHARD = 2
TEST_SUITE = 'cms-unit'
}
steps {
script {
runPythonTests()
}
}
post {
always {
script {
savePythonTestArtifacts()
}
}
}
}
stage('commonlib-unit-1') {
agent { label "jenkins-worker" }
environment {
SHARD = 1
TEST_SUITE = 'commonlib-unit'
}
steps {
script {
runPythonTests()
}
}
post {
always {
script {
savePythonTestArtifacts()
}
}
}
}
stage('commonlib-unit-2') {
agent { label "jenkins-worker" }
environment {
SHARD = 2
TEST_SUITE = 'commonlib-unit'
}
steps {
script {
runPythonTests()
}
}
post {
always {
script {
savePythonTestArtifacts()
}
}
}
}
stage('commonlib-unit-3') {
agent { label "jenkins-worker" }
environment {
SHARD = 3
TEST_SUITE = 'commonlib-unit'
}
steps {
script {
runPythonTests()
}
}
post {
always {
script {
savePythonTestArtifacts()
pythonTestCleanup()
}
}
}
@@ -327,30 +105,16 @@ pipeline {
SUBSET_JOB = "null" // Keep this variable until we can remove the $SUBSET_JOB path from .coveragerc
}
steps {
ansiColor('gnome-terminal') {
sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) {
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${sha1}']],
doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [],
userRemoteConfigs: [[credentialsId: 'jenkins-worker',
refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*',
url: 'git@github.com:edx/edx-platform.git']]]
unstash 'lms-unit-1-reports'
unstash 'lms-unit-2-reports'
unstash 'lms-unit-3-reports'
unstash 'lms-unit-4-reports'
unstash 'lms-unit-5-reports'
unstash 'lms-unit-6-reports'
unstash 'lms-unit-7-reports'
unstash 'lms-unit-8-reports'
unstash 'lms-unit-9-reports'
unstash 'lms-unit-10-reports'
unstash 'cms-unit-1-reports'
unstash 'cms-unit-2-reports'
unstash 'commonlib-unit-1-reports'
unstash 'commonlib-unit-2-reports'
unstash 'commonlib-unit-3-reports'
sh "./scripts/jenkins-report.sh"
}
sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) {
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${ghprbActualCommit}']],
doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [],
userRemoteConfigs: [[credentialsId: 'jenkins-worker',
refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*',
url: 'git@github.com:edx/edx-platform.git']]]
unstash 'lms-unit-reports'
unstash 'cms-unit-reports'
unstash 'commonlib-unit-reports'
sh "./scripts/jenkins-report.sh"
}
}
post {

163
pavelib/paver_tests/test_paver_pytest_cmds.py Normal file
View File

@@ -0,0 +1,163 @@
"""
Tests for the pytest paver commands themselves.
Run just this test with: paver test_lib -t pavelib/paver_tests/test_paver_pytest_cmds.py
"""
import unittest
import os
import ddt
from pavelib.utils.test.suites import SystemTestSuite, LibTestSuite
from pavelib.utils.envs import Env
XDIST_TESTING_IP_ADDRESS_LIST = '0.0.0.1,0.0.0.2,0.0.0.3'
@ddt.ddt
class TestPaverPytestCmd(unittest.TestCase):
"""
Test Paver pytest commands
"""
def _expected_command(self, root, test_id, pytestSubclass, run_under_coverage=True,
processes=0, xdist_ip_addresses=None):
"""
Returns the command that is expected to be run for the given test spec
and store.
"""
report_dir = Env.REPORT_DIR / root
shard = os.environ.get('SHARD')
if shard:
report_dir = report_dir / 'shard_' + shard
expected_statement = [
"python",
"-Wd",
"-m",
"pytest"
]
if pytestSubclass == "SystemTestSuite":
expected_statement.append("--ds={}".format('{}.envs.{}'.format(root, Env.TEST_SETTINGS)))
expected_statement.append("--junitxml={}".format(report_dir / "nosetests.xml"))
if xdist_ip_addresses:
expected_statement.append('--dist=loadscope')
for ip in xdist_ip_addresses.split(','):
if processes <= 0:
processes = 1
if pytestSubclass == "SystemTestSuite":
django_env_var_cmd = "export DJANGO_SETTINGS_MODULE={}.envs.test".format(root)
elif pytestSubclass == "LibTestSuite":
if 'pavelib/paver_tests' in test_id:
django_env_var_cmd = "export DJANGO_SETTINGS_MODULE={}.envs.test".format(root)
else:
django_env_var_cmd = "export DJANGO_SETTINGS_MODULE='openedx.tests.settings'"
xdist_string = '--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \
'//python="source /edx/app/edxapp/edxapp_env; {}; python"' \
'//chdir="/edx/app/edxapp/edx-platform"' \
.format(processes, ip, django_env_var_cmd)
expected_statement.append(xdist_string)
for rsync_dir in Env.rsync_dirs():
expected_statement.append('--rsyncdir {}'.format(rsync_dir))
else:
if processes == -1:
expected_statement.append('-n auto')
expected_statement.append('--dist=loadscope')
elif processes != 0:
expected_statement.append('-n {}'.format(processes))
expected_statement.append('--dist=loadscope')
expected_statement.extend([
"-p no:randomly",
test_id
])
if run_under_coverage:
if xdist_ip_addresses:
for module in Env.covered_modules():
expected_statement.append('--cov')
expected_statement.append(module)
else:
expected_statement.append('--cov')
expected_statement.append('--cov-report=')
return expected_statement
@ddt.data('lms', 'cms')
def test_SystemTestSuite_suites(self, system):
test_id = 'tests'
suite = SystemTestSuite(system, test_id=test_id)
assert suite.cmd == self._expected_command(system, test_id, "SystemTestSuite")
@ddt.data('lms', 'cms')
def test_SystemTestSuite_auto_processes(self, system):
test_id = 'tests'
suite = SystemTestSuite(system, test_id=test_id, processes=-1)
assert suite.cmd == self._expected_command(system, test_id, "SystemTestSuite", processes=-1)
@ddt.data('lms', 'cms')
def test_SystemTestSuite_multi_processes(self, system):
test_id = 'tests'
suite = SystemTestSuite(system, test_id=test_id, processes=3)
assert suite.cmd == self._expected_command(system, test_id, "SystemTestSuite", processes=3)
@ddt.data('lms', 'cms')
def test_SystemTestSuite_with_xdist(self, system):
test_id = 'tests'
suite = SystemTestSuite(system, test_id=test_id, xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
assert suite.cmd == self._expected_command(system, test_id, "SystemTestSuite",
xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
@ddt.data('lms', 'cms')
def test_SystemTestSuite_with_xdist_multi_processes(self, system):
test_id = 'tests'
suite = SystemTestSuite(system, test_id=test_id, processes=2, xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
assert suite.cmd == self._expected_command(system, test_id, "SystemTestSuite", processes=2,
xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
@ddt.data('lms', 'cms')
def test_SystemTestSuite_with_xdist_negative_processes(self, system):
test_id = 'tests'
suite = SystemTestSuite(system, test_id=test_id, processes=-1, xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
assert suite.cmd == self._expected_command(system, test_id, "SystemTestSuite", processes=-1,
xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
@ddt.data('common/lib/xmodule', 'pavelib/paver_tests')
def test_LibTestSuite_suites(self, system):
test_id = 'tests'
suite = LibTestSuite(system, test_id=test_id)
assert suite.cmd == self._expected_command(system, test_id, "LibTestSuite")
@ddt.data('common/lib/xmodule', 'pavelib/paver_tests')
def test_LibTestSuite_auto_processes(self, system):
test_id = 'tests'
suite = LibTestSuite(system, test_id=test_id, processes=-1)
assert suite.cmd == self._expected_command(system, test_id, "LibTestSuite", processes=-1)
@ddt.data('common/lib/xmodule', 'pavelib/paver_tests')
def test_LibTestSuite_multi_processes(self, system):
test_id = 'tests'
suite = LibTestSuite(system, test_id=test_id, processes=3)
assert suite.cmd == self._expected_command(system, test_id, "LibTestSuite", processes=3)
@ddt.data('common/lib/xmodule', 'pavelib/paver_tests')
def test_LibTestSuite_with_xdist(self, system):
test_id = 'tests'
suite = LibTestSuite(system, test_id=test_id, xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
assert suite.cmd == self._expected_command(system, test_id, "LibTestSuite",
xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
@ddt.data('common/lib/xmodule', 'pavelib/paver_tests')
def test_LibTestSuite_with_xdist_multi_processes(self, system):
test_id = 'tests'
suite = LibTestSuite(system, test_id=test_id, processes=2, xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
assert suite.cmd == self._expected_command(system, test_id, "LibTestSuite", processes=2,
xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
@ddt.data('common/lib/xmodule', 'pavelib/paver_tests')
def test_LibTestSuite_with_xdist_negative_processes(self, system):
test_id = 'tests'
suite = LibTestSuite(system, test_id=test_id, processes=-1, xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)
assert suite.cmd == self._expected_command(system, test_id, "LibTestSuite", processes=-1,
xdist_ip_addresses=XDIST_TESTING_IP_ADDRESS_LIST)

8
pavelib/tests.py
View File

@@ -72,7 +72,7 @@ __test__ = False # do not collect
make_option(
'--xdist_ip_addresses',
dest='xdist_ip_addresses',
help="Space separated string of ip addresses to shard tests to via xdist."
help="Comma separated string of ip addresses to shard tests to via xdist."
)
], share_with=['pavelib.utils.test.utils.clean_reports_dir'])
@PassthroughTask
@@ -160,8 +160,10 @@ def test_system(options, passthrough_options):
make_option(
'--xdist_ip_addresses',
dest='xdist_ip_addresses',
help="Space separated string of ip addresses to shard tests to via xdist."
)
help="Comma separated string of ip addresses to shard tests to via xdist."
),
make_option('-p', '--processes', dest='processes', default=0, help='number of processes to use running tests'),
make_option('-r', '--randomize', action='store_true', help='run the tests in a random order'),
], share_with=['pavelib.utils.test.utils.clean_reports_dir'])
@PassthroughTask
@timed

52
pavelib/utils/test/suites/pytest_suite.py
View File

@@ -158,12 +158,20 @@ class SystemTestSuite(PytestSuite):
if self.disable_capture:
cmd.append("-s")
if self.xdist_ip_addresses:
cmd.append('--dist=loadscope')
for ip in self.xdist_ip_addresses.split(' '):
xdist_string = '--tx ssh=ubuntu@{}//python="source /edx/app/edxapp/edxapp_env; ' \
'python"//chdir="/edx/app/edxapp/edx-platform"'.format(ip)
if self.processes <= 0:
xdist_remote_processes = 1
else:
xdist_remote_processes = self.processes
for ip in self.xdist_ip_addresses.split(','):
# The django settings runtime command does not propagate to xdist remote workers
django_env_var_cmd = 'export DJANGO_SETTINGS_MODULE={}' \
.format('{}.envs.{}'.format(self.root, self.settings))
xdist_string = '--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \
'//python="source /edx/app/edxapp/edxapp_env; {}; python"' \
'//chdir="/edx/app/edxapp/edx-platform"' \
.format(xdist_remote_processes, ip, django_env_var_cmd)
cmd.append(xdist_string)
for rsync_dir in Env.rsync_dirs():
cmd.append('--rsyncdir {}'.format(rsync_dir))
@@ -239,6 +247,14 @@ class LibTestSuite(PytestSuite):
self.test_id = kwargs.get('test_id', self.root)
self.eval_attr = kwargs.get('eval_attr', None)
self.xdist_ip_addresses = kwargs.get('xdist_ip_addresses', None)
self.randomize = kwargs.get('randomize', None)
self.processes = kwargs.get('processes', None)
if self.processes is None:
# Don't use multiprocessing by default
self.processes = 0
self.processes = int(self.processes)
@property
def cmd(self):
@@ -251,8 +267,6 @@ class LibTestSuite(PytestSuite):
'-Wd',
'-m',
'pytest',
'-p',
'no:randomly',
'--junitxml={}'.format(self.xunit_report),
])
cmd.extend(self.passthrough_options + self.test_options_flags)
@@ -265,13 +279,33 @@ class LibTestSuite(PytestSuite):
if self.xdist_ip_addresses:
cmd.append('--dist=loadscope')
for ip in self.xdist_ip_addresses.split(' '):
xdist_string = '--tx ssh=ubuntu@{}//python="source /edx/app/edxapp/edxapp_env; ' \
'python"//chdir="/edx/app/edxapp/edx-platform"'.format(ip)
if self.processes <= 0:
xdist_remote_processes = 1
else:
xdist_remote_processes = self.processes
for ip in self.xdist_ip_addresses.split(','):
# The django settings runtime command does not propagate to xdist remote workers
if 'pavelib/paver_tests' in self.test_id:
django_env_var_cmd = "export DJANGO_SETTINGS_MODULE='lms.envs.test'"
else:
django_env_var_cmd = "export DJANGO_SETTINGS_MODULE='openedx.tests.settings'"
xdist_string = '--tx {}*ssh="ubuntu@{} -o StrictHostKeyChecking=no"' \
'//python="source /edx/app/edxapp/edxapp_env; {}; python"' \
'//chdir="/edx/app/edxapp/edx-platform"' \
.format(xdist_remote_processes, ip, django_env_var_cmd)
cmd.append(xdist_string)
for rsync_dir in Env.rsync_dirs():
cmd.append('--rsyncdir {}'.format(rsync_dir))
else:
if self.processes == -1:
cmd.append('-n auto')
cmd.append('--dist=loadscope')
elif self.processes != 0:
cmd.append('-n {}'.format(self.processes))
cmd.append('--dist=loadscope')
if not self.randomize:
cmd.append("-p no:randomly")
if self.eval_attr:
cmd.append("-a '{}'".format(self.eval_attr))

20
scripts/unit-tests.sh
View File

@@ -29,8 +29,6 @@ set -e
#
###############################################################################
PAVER_ARGS="-v"
PARALLEL="--processes=-1"
export SKIP_NPM_INSTALL="True"
# Skip re-installation of Python prerequisites inside a tox execution.
@@ -38,6 +36,20 @@ if [[ -n "$TOXENV" ]]; then
export NO_PREREQ_INSTALL="True"
fi
if [[ -n "$XDIST_NUM_TASKS" ]]; then
bash scripts/xdist/prepare_xdist_nodes.sh
PAVER_ARGS="-v --xdist_ip_addresses="$(<pytest_task_ips.txt)""
export SHARD="all"
if [[ -n "$XDIST_REMOTE_NUM_PROCESSES" ]]; then
PARALLEL="--processes=$XDIST_REMOTE_NUM_PROCESSES"
else
PARALLEL="--processes=1"
fi
else
PAVER_ARGS="-v"
PARALLEL="--processes=-1"
fi
case "${TEST_SUITE}" in
"lms-unit")
@@ -65,7 +77,7 @@ case "${TEST_SUITE}" in
"cms-unit")
case "$SHARD" in
"all")
paver test_system -s cms --disable_capture ${PAVER_ARGS} 2> cms-tests.log
paver test_system -s cms --disable_capture ${PAVER_ARGS} ${PARALLEL} 2> cms-tests.log
;;
1)
paver test_system -s cms --disable_capture --eval-attr="shard==$SHARD" ${PAVER_ARGS} 2> cms-tests.${SHARD}.log
@@ -87,7 +99,7 @@ case "${TEST_SUITE}" in
"commonlib-unit")
case "$SHARD" in
"all")
paver test_lib --disable_capture ${PAVER_ARGS} 2> common-tests.log
paver test_lib --disable_capture ${PAVER_ARGS} ${PARALLEL} 2> common-tests.log
;;
[1-2])
paver test_lib -l common/lib/xmodule --disable_capture --eval-attr="shard==$SHARD" ${PAVER_ARGS} 2> common-tests.${SHARD}.log

6
scripts/xdist/prepare_xdist_nodes.sh
View File

@@ -1,16 +1,16 @@
#!/bin/bash
set -e
echo "Spinning up xdist containers with pytest_container_manager.py"
python scripts/xdist/pytest_container_manager.py -a up -n ${XDIST_NUM_TASKS} \
-t ${XDIST_CONTAINER_TASK_NAME} \
-s ${XDIST_CONTAINER_SUBNET} \
-sg ${XDIST_CONTAINER_SECURITY_GROUP}
ip_list=$(<pytest_task_ips.txt)
for ip in $ip_list
for ip in $(echo $ip_list | sed "s/,/ /g")
do
container_reqs_cmd="ssh ubuntu@$ip 'cd /edx/app/edxapp/edx-platform;
container_reqs_cmd="ssh -o StrictHostKeyChecking=no ubuntu@$ip 'cd /edx/app/edxapp/edx-platform;
git pull -q; git checkout -q ${XDIST_GIT_BRANCH};
source /edx/app/edxapp/edxapp_env; pip install -qr requirements/edx/testing.txt' & "

36
scripts/xdist/pytest_container_manager.py
View File

@@ -104,13 +104,13 @@ class PytestContainerManager():
logger.info("Successfully booted up {} tasks.".format(number_of_tasks))
# Generate .txt files containing IP addresses and task arns
ip_list_string = " ".join(ip_addresses)
ip_list_string = ",".join(ip_addresses)
logger.info("Task IP list: {}".format(ip_list_string))
ip_list_file = open("pytest_task_ips.txt", "w")
ip_list_file.write(ip_list_string)
ip_list_file.close()
task_arn_list_string = " ".join(task_arns)
task_arn_list_string = ",".join(task_arns)
logger.info("Task arn list: {}".format(task_arn_list_string))
task_arn_file = open("pytest_task_arns.txt", "w")
task_arn_file.write(task_arn_list_string)
@@ -120,7 +120,7 @@ class PytestContainerManager():
"""
Terminates tasks based on a list of task_arns.
"""
for task_arn in task_arns:
for task_arn in task_arns.split(','):
response = self.ecs.stop_task(
cluster=self.cluster_name,
task=task_arn,
@@ -134,23 +134,26 @@ if __name__ == "__main__":
description="PytestContainerManager, manages ECS tasks in an AWS cluster."
)
parser.add_argument('--region', '-g', default='us-east-1',
help="AWS region where ECS infrastructure lives. Defaults to us-east-1")
parser.add_argument('--action', '-a', choices=['up', 'down'], default=None,
help="Action for PytestContainerManager to perform. "
"Either up for spinning up AWS ECS tasks or down for stopping them")
parser.add_argument('--cluster', '-c', default="jenkins-worker-containers",
help="AWS Cluster name where the tasks run. Defaults to"
"the testeng cluster: jenkins-worker-containers")
parser.add_argument('--action', '-a', choices=['up', 'down'], default=None,
help="Action for PytestContainerManager to perform. "
"Either up for spinning up AWS ECS tasks or down for stopping them")
parser.add_argument('--region', '-g', default='us-east-1',
help="AWS region where ECS infrastructure lives. Defaults to us-east-1")
# Spinning up tasks
parser.add_argument('--launch_type', default='FARGATE', choices=['EC2', 'FARGATE'],
help="ECS launch type for tasks. Defaults to FARGATE")
parser.add_argument('--num_tasks', '-n', type=int, default=None,
help="Number of ECS tasks to spin up")
parser.add_argument('--task_name', '-t', default=None,
help="Name of the task definition")
parser.add_argument('--public_ip', choices=['ENABLED', 'DISABLED'],
default='DISABLED', help="Whether the tasks should have a public IP")
parser.add_argument('--subnets', '-s', nargs='+', default=None,
help="List of subnets for the tasks to exist in")
@@ -158,19 +161,16 @@ if __name__ == "__main__":
parser.add_argument('--security_groups', '-sg', nargs='+', default=None,
help="List of security groups to apply to the tasks")
parser.add_argument('--public_ip', choices=['ENABLED', 'DISABLED'],
default='DISABLED', help="Whether the tasks should have a public IP")
parser.add_argument('--launch_type', default='FARGATE', choices=['EC2', 'FARGATE'],
help="ECS launch type for tasks. Defaults to FARGATE")
parser.add_argument('--task_name', '-t', default=None,
help="Name of the task definition")
# Terminating tasks
parser.add_argument('--task_arns', '-arns', nargs='+', default=None,
help="Task arns to terminate")
parser.add_argument('--reason', '-r', default="Finished executing tests",
help="Reason for terminating tasks")
parser.add_argument('--task_arns', '-arns', default=None,
help="Task arns to terminate")
args = parser.parse_args()
containerManager = PytestContainerManager(args.region, args.cluster)

10
scripts/xdist/terminate_xdist_nodes.sh Normal file
View File

@@ -0,0 +1,10 @@
#!/bin/bash
set -e
if [ -f pytest_task_arns.txt ]; then
echo "Terminating xdist containers with pytest_container_manager.py"
xdist_task_arns=$(<pytest_task_arns.txt)
python scripts/xdist/pytest_container_manager.py -a down --task_arns ${xdist_task_arns}
else
echo "File: pytest_task_arns.txt not found"
fi

1
tox.ini
View File

@@ -51,6 +51,7 @@ passenv =
XDIST_CONTAINER_TASK_NAME
XDIST_GIT_BRANCH
XDIST_NUM_TASKS
XDIST_REMOTE_NUM_PROCESSES
deps =
django18: Django>=1.8,<1.9
django19: Django>=1.9,<1.10