diff --git a/Jenkinsfile b/Jenkinsfile index f0f69cae08..ce1ac7ff69 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,9 +1,9 @@ def runPythonTests() { sshagent(credentials: ['jenkins-worker', 'jenkins-worker-pem'], ignoreMissing: true) { checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${ghprbActualCommit}']], - doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], - userRemoteConfigs: [[credentialsId: 'jenkins-worker', - refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*', + doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', honorRefspec: true, + noTags: true, shallow: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker', + refspec: '+refs/heads/master:refs/remotes/origin/master +refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*', url: 'git@github.com:edx/edx-platform.git']]] console_output = sh(returnStdout: true, script: 'bash scripts/all-tests.sh').trim() dir('stdout') { @@ -107,9 +107,10 @@ pipeline { steps { sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) { checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${ghprbActualCommit}']], - doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], + doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', + honorRefspec: true, noTags: true, shallow: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker', - refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*', + refspec: '+refs/heads/master:refs/remotes/origin/master +refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*', url: 'git@github.com:edx/edx-platform.git']]] unstash 'lms-unit-reports' unstash 'cms-unit-reports' diff --git a/scripts/Jenkinsfiles/python b/scripts/Jenkinsfiles/python new file mode 100644 index 0000000000..ce1ac7ff69 --- /dev/null +++ b/scripts/Jenkinsfiles/python @@ -0,0 +1,133 @@ +def runPythonTests() { + sshagent(credentials: ['jenkins-worker', 'jenkins-worker-pem'], ignoreMissing: true) { + checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${ghprbActualCommit}']], + doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', honorRefspec: true, + noTags: true, shallow: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker', + refspec: '+refs/heads/master:refs/remotes/origin/master +refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*', + url: 'git@github.com:edx/edx-platform.git']]] + console_output = sh(returnStdout: true, script: 'bash scripts/all-tests.sh').trim() + dir('stdout') { + writeFile file: "${TEST_SUITE}-stdout.log", text: console_output + } + stash includes: 'reports/**/*coverage*', name: "${TEST_SUITE}-reports" + } +} + +def pythonTestCleanup() { + archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/**/*,test_root/log/**/*.log,**/nosetests.xml,stdout/*.log,*.log' + junit '**/nosetests.xml' + sh '''source $HOME/edx-venv/bin/activate + bash scripts/xdist/terminate_xdist_nodes.sh''' +} + +pipeline { + agent { label "coverage-worker" } + options { + timestamps() + timeout(60) + } + environment { + XDIST_CONTAINER_SUBNET = credentials('XDIST_CONTAINER_SUBNET') + XDIST_CONTAINER_SECURITY_GROUP = credentials('XDIST_CONTAINER_SECURITY_GROUP') + XDIST_CONTAINER_TASK_NAME = "jenkins-worker-task" + XDIST_GIT_BRANCH = "${ghprbActualCommit}" + } + stages { + stage('Run Tests') { + parallel { + stage("lms-unit") { + agent { label "jenkins-worker" } + environment { + TEST_SUITE = "lms-unit" + XDIST_NUM_TASKS = 10 + XDIST_REMOTE_NUM_PROCESSES = 2 + } + steps { + script { + runPythonTests() + } + } + post { + always { + script { + pythonTestCleanup() + } + } + } + } + stage("cms-unit") { + agent { label "jenkins-worker" } + environment { + TEST_SUITE = "cms-unit" + XDIST_NUM_TASKS = 4 + XDIST_REMOTE_NUM_PROCESSES = 2 + } + steps { + script { + runPythonTests() + } + } + post { + always { + script { + pythonTestCleanup() + } + } + } + } + stage("commonlib-unit") { + agent { label "jenkins-worker" } + environment { + TEST_SUITE = "commonlib-unit" + XDIST_NUM_TASKS = 3 + XDIST_REMOTE_NUM_PROCESSES = 2 + } + steps { + script { + runPythonTests() + } + } + post { + always { + script { + pythonTestCleanup() + } + } + } + } + } + } + stage('Run coverage') { + environment { + CODE_COV_TOKEN = credentials('CODE_COV_TOKEN') + TARGET_BRANCH = "origin/master" + CI_BRANCH = "${ghprbSourceBranch}" + SUBSET_JOB = "null" // Keep this variable until we can remove the $SUBSET_JOB path from .coveragerc + } + steps { + sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) { + checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${ghprbActualCommit}']], + doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', + honorRefspec: true, noTags: true, shallow: true]], submoduleCfg: [], + userRemoteConfigs: [[credentialsId: 'jenkins-worker', + refspec: '+refs/heads/master:refs/remotes/origin/master +refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*', + url: 'git@github.com:edx/edx-platform.git']]] + unstash 'lms-unit-reports' + unstash 'cms-unit-reports' + unstash 'commonlib-unit-reports' + sh "./scripts/jenkins-report.sh" + } + } + post { + always { + publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true, + reportDir: 'reports', reportFiles: 'diff_coverage_combined.html', + reportName: 'Diff Coverage Report', reportTitles: '']) + publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true, + reportDir: 'reports/cover', reportFiles: 'index.html', + reportName: 'Coverage.py Report', reportTitles: '']) + } + } + } + } +}