Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix Jenkins virtualenv cleanup (#22356)

Browse Source
This commit is contained in:
Jeremy Bowman
2019-11-21 11:05:05 -05:00
committed by GitHub
parent 889daa8baf
commit 052f853ec1
5 changed files with 16 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
pavelib/utils/envs.py
View File

@@ -78,6 +78,9 @@ class Env(object):
"lib" / "custom_a11y_rules.js"
)
# Which Python version should be used in xdist workers?
PYTHON_VERSION = os.environ.get("PYTHON_VERSION", "2.7")
# If set, put reports for run in "unique" directories.
# The main purpose of this is to ensure that the reports can be 'slurped'
# in the main jenkins flow job without overwriting the reports from other

8
pavelib/utils/test/suites/pytest_suite.py
View File

@@ -184,9 +184,9 @@ class SystemTestSuite(PytestSuite):
.format('{}.envs.{}'.format(self.root, self.settings),
self.disable_courseenrollment_history)
xdist_string = u'--tx {}*ssh="jenkins@{} -o StrictHostKeyChecking=no"' \
'//python="source edx-venv/bin/activate; {}; python"' \
'//python="source edx-venv-{}/edx-venv/bin/activate; {}; python"' \
'//chdir="edx-platform"' \
.format(xdist_remote_processes, ip, env_var_cmd)
.format(xdist_remote_processes, ip, Env.PYTHON_VERSION, env_var_cmd)
cmd.append(xdist_string)
for rsync_dir in Env.rsync_dirs():
cmd.append(u'--rsyncdir {}'.format(rsync_dir))
@@ -310,9 +310,9 @@ class LibTestSuite(PytestSuite):
.format(django_env_var_cmd, self.disable_courseenrollment_history)
xdist_string = u'--tx {}*ssh="jenkins@{} -o StrictHostKeyChecking=no"' \
'//python="source edx-venv/bin/activate; {}; python"' \
'//python="source edx-venv-{}/edx-venv/bin/activate; {}; python"' \
'//chdir="edx-platform"' \
.format(xdist_remote_processes, ip, env_var_cmd)
.format(xdist_remote_processes, ip, Env.PYTHON_VERSION, env_var_cmd)
cmd.append(xdist_string)
for rsync_dir in Env.rsync_dirs():
cmd.append(u'--rsyncdir {}'.format(rsync_dir))

2
scripts/Jenkinsfiles/python
View File

@@ -16,7 +16,7 @@ def pythonTestCleanup() {
archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/**/*,test_root/log/**/*.log,**/nosetests.xml,*.log'
sendSplunkFile excludes: '', includes: '**/timing*.log', sizeLimit: '10MB'
junit '**/nosetests.xml'
sh '''source $HOME/edx-venv/bin/activate
sh '''source $HOME/edx-venv-$PYTHON_VERSION/edx-venv/bin/activate
bash scripts/xdist/terminate_xdist_nodes.sh'''
}

8
scripts/jenkins-common.sh
View File

@@ -20,14 +20,16 @@ fi
# Reset the jenkins worker's virtualenv back to the
# state it was in when the instance was spun up.
if [ -z ${PYTHON_VERSION+x} ] || [[ ${PYTHON_VERSION} == 'null' ]]; then
VENV_PATH="$HOME"
ARCHIVED_VENV="edx-venv_clean.tar.gz"
else
VENV_PATH="$HOME/edx-venv-$PYTHON_VERSION"
ARCHIVED_VENV="edx-venv_clean-$PYTHON_VERSION.tar.gz"
fi
if [ -e $HOME/$ARCHIVED_VENV ]; then
rm -rf $HOME/edx-venv
tar -C $HOME -xf $HOME/$ARCHIVED_VENV
rm -rf $VENV_PATH/edx-venv
tar -C $VENV_PATH -xf $HOME/$ARCHIVED_VENV
fi
# Load the npm packages from the time the worker was built
@@ -41,7 +43,7 @@ if [ -e $HOME/edx-npm-cache_clean.tar.gz ]; then
fi
# Activate the Python virtualenv
source $HOME/edx-venv/bin/activate
source $VENV_PATH/edx-venv/bin/activate
# add the node packages dir to PATH
PATH=$PATH:node_modules/.bin

6
scripts/xdist/prepare_xdist_nodes.sh
View File

@@ -23,9 +23,9 @@ do
worker_reqs_cmd="ssh -o StrictHostKeyChecking=no jenkins@$ip
'git clone --branch master --depth 1 -q https://github.com/edx/edx-platform.git; cd edx-platform;
git fetch -fq origin ${XDIST_GIT_REFSPEC}; git checkout -q ${XDIST_GIT_BRANCH};
rm -rf /home/jenkins/edx-venv;
tar -C /home/jenkins -xf /home/jenkins/edx-venv_clean-${PYTHON_VERSION}.tar.gz;
source ../edx-venv/bin/activate;
rm -rf /home/jenkins/edx-venv-${PYTHON_VERSION}/edx-venv;
tar -C /home/jenkins/edx-venv-${PYTHON_VERSION} -xf /home/jenkins/edx-venv_clean-${PYTHON_VERSION}.tar.gz;
source ../edx-venv-${PYTHON_VERSION}/edx-venv/bin/activate;
pip install -q ${DJANGO_REQUIREMENT} -r requirements/edx/testing.txt; mkdir reports' & "
cmd=$cmd$worker_reqs_cmd