TE-2804 Optimize pipeline git usage

2018-11-07 17:25:57 -05:00
parent db88e457a3
commit 84d780e681
2 changed files with 139 additions and 5 deletions
--- a/scripts/Jenkinsfiles/python
+++ b/scripts/Jenkinsfiles/python
@@ -0,0 +1,133 @@
+def runPythonTests() {
+    sshagent(credentials: ['jenkins-worker', 'jenkins-worker-pem'], ignoreMissing: true) {
+        checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${ghprbActualCommit}']],
+            doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', honorRefspec: true,
+            noTags: true, shallow: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker',
+            refspec: '+refs/heads/master:refs/remotes/origin/master +refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*',
+            url: 'git@github.com:edx/edx-platform.git']]]
+        console_output = sh(returnStdout: true, script: 'bash scripts/all-tests.sh').trim()
+        dir('stdout') {
+            writeFile file: "${TEST_SUITE}-stdout.log", text: console_output
+        }
+        stash includes: 'reports/**/*coverage*', name: "${TEST_SUITE}-reports"
+    }
+}
+
+def pythonTestCleanup() {
+    archiveArtifacts allowEmptyArchive: true, artifacts: 'reports/**/*,test_root/log/**/*.log,**/nosetests.xml,stdout/*.log,*.log'
+    junit '**/nosetests.xml'
+    sh '''source $HOME/edx-venv/bin/activate
+    bash scripts/xdist/terminate_xdist_nodes.sh'''
+}
+
+pipeline {
+    agent { label "coverage-worker" }
+    options {
+        timestamps()
+        timeout(60)
+    }
+    environment {
+        XDIST_CONTAINER_SUBNET = credentials('XDIST_CONTAINER_SUBNET')
+        XDIST_CONTAINER_SECURITY_GROUP = credentials('XDIST_CONTAINER_SECURITY_GROUP')
+        XDIST_CONTAINER_TASK_NAME = "jenkins-worker-task"
+        XDIST_GIT_BRANCH = "${ghprbActualCommit}"
+    }
+    stages {
+        stage('Run Tests') {
+            parallel {
+                stage("lms-unit") {
+                    agent { label "jenkins-worker" }
+                    environment {
+                        TEST_SUITE = "lms-unit"
+                        XDIST_NUM_TASKS = 10
+                        XDIST_REMOTE_NUM_PROCESSES = 2
+                    }
+                    steps {
+                        script {
+                            runPythonTests()
+                        }
+                    }
+                    post {
+                        always {
+                            script {
+                                pythonTestCleanup()
+                            }
+                        }
+                    }
+                }
+                stage("cms-unit") {
+                    agent { label "jenkins-worker" }
+                    environment {
+                        TEST_SUITE = "cms-unit"
+                        XDIST_NUM_TASKS = 4
+                        XDIST_REMOTE_NUM_PROCESSES = 2
+                    }
+                    steps {
+                        script {
+                            runPythonTests()
+                        }
+                    }
+                    post {
+                        always {
+                            script {
+                                pythonTestCleanup()
+                            }
+                        }
+                    }
+                }
+                stage("commonlib-unit") {
+                    agent { label "jenkins-worker" }
+                    environment {
+                        TEST_SUITE = "commonlib-unit"
+                        XDIST_NUM_TASKS = 3
+                        XDIST_REMOTE_NUM_PROCESSES = 2
+                    }
+                    steps {
+                        script {
+                            runPythonTests()
+                        }
+                    }
+                    post {
+                        always {
+                            script {
+                                pythonTestCleanup()
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        stage('Run coverage') {
+            environment {
+                CODE_COV_TOKEN = credentials('CODE_COV_TOKEN')
+                TARGET_BRANCH = "origin/master"
+                CI_BRANCH = "${ghprbSourceBranch}"
+                SUBSET_JOB = "null" // Keep this variable until we can remove the $SUBSET_JOB path from .coveragerc
+            }
+            steps {
+                sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) {
+                    checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${ghprbActualCommit}']],
+                        doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption',
+                        honorRefspec: true, noTags: true, shallow: true]], submoduleCfg: [],
+                        userRemoteConfigs: [[credentialsId: 'jenkins-worker',
+                        refspec: '+refs/heads/master:refs/remotes/origin/master +refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*',
+                        url: 'git@github.com:edx/edx-platform.git']]]
+                    unstash 'lms-unit-reports'
+                    unstash 'cms-unit-reports'
+                    unstash 'commonlib-unit-reports'
+                    sh "./scripts/jenkins-report.sh"
+                }
+            }
+            post {
+                always {
+                    publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true,
+                        reportDir: 'reports', reportFiles: 'diff_coverage_combined.html',
+                        reportName: 'Diff Coverage Report', reportTitles: ''])
+                    publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true,
+                        reportDir: 'reports/cover', reportFiles: 'index.html',
+                        reportName: 'Coverage.py Report', reportTitles: ''])
+                }
+            }
+        }
+    }
+}