Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow merge builds for lettuce and python

Browse Source
This commit is contained in:
Michael Youngstrom
2019-01-02 16:44:54 -05:00
parent c1e38a19cb
commit c2f57f975b
3 changed files with 203 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
scripts/Jenkinsfiles/bokchoy
View File

@@ -1,7 +1,6 @@
def runBokchoyTests() {
// Determine git refspec, branch, and clone type
git_shallow_clone = true
git_extensions = []
if (env.ghprbActualCommit) {
git_branch = "${ghprbActualCommit}"
git_refspec = "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"

88
scripts/Jenkinsfiles/lettuce
View File

@@ -1,10 +1,17 @@
def runLettuceTests() {
// Determine git refspec, branch, and clone type
if (env.ghprbActualCommit) {
git_branch = "${ghprbActualCommit}"
git_refspec = "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
} else {
git_branch = 'master'
git_refspec = "+refs/heads/master:refs/remotes/origin/master"
}
sshagent(credentials: ['jenkins-worker', 'jenkins-worker-pem'], ignoreMissing: true) {
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${ghprbActualCommit}']],
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: git_branch]],
doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', honorRefspec: true,
noTags: true, shallow: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker',
refspec: '+refs/heads/master:refs/remotes/origin/master +refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*',
url: 'git@github.com:edx/edx-platform.git']]]
refspec: git_refspec, url: 'git@github.com:edx/edx-platform.git']]]
console_output = sh(returnStdout: true, script: 'bash scripts/all-tests.sh').trim()
dir('stdout') {
writeFile file: "${TEST_SUITE}-stdout.log", text: console_output
@@ -18,12 +25,32 @@ def lettuceTestCleanup() {
}
pipeline {
agent none
agent { label "jenkins-worker" }
options {
timestamps()
timeout(60)
}
stages {
stage('Mark build as pending on Github') {
when {
// Only run github-build-status for master builds
expression { env.ghprbActualCommit == null }
}
steps {
script {
commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
build job: 'github-build-status',
parameters: [
string(name: 'GIT_SHA', value: commit_sha),
string(name: 'GITHUB_ORG', value: 'edx'), string(name: 'GITHUB_REPO', value: 'edx-platform'),
string(name: 'TARGET_URL', value: "${BUILD_URL}"),
string(name: 'DESCRIPTION', value: 'Pending'), string(name: 'CONTEXT', value: 'jenkins/lettuce'),
string(name: 'CREATE_DEPLOYMENT', value: 'false'), string(name: 'BUILD_STATUS', value: 'pending')
],
propagate: false, wait: false
}
}
}
stage('Run Tests') {
parallel {
stage("lms-acceptance") {
@@ -65,4 +92,57 @@ pipeline {
}
}
}
post {
always {
script{
if (env.ghprbPullId != null) {
// For PR jobs, run the edx-platform-test-notifier for PR reporting
build job: 'edx-platform-test-notifier', parameters: [string(name: 'PR_NUMBER', value: "${ghprbPullId}")], wait: false
} else {
// For master jobs run github-build-status and report to slack when necessary
if (currentBuild.currentResult == "SUCCESS") {
create_deployment = "true"
build_status = "success"
build_description = "Build Passed"
}
else {
create_deployment = "false"
build_status = "failure"
build_description = "Build Failed"
}
commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
build job: 'github-build-status',
parameters: [
string(name: 'GIT_SHA', value: commit_sha),
string(name: 'GITHUB_ORG', value: 'edx'), string(name: 'GITHUB_REPO', value: 'edx-platform'),
string(name: 'TARGET_URL', value: "${BUILD_URL}"),
string(name: 'DESCRIPTION', value: build_description), string(name: 'CONTEXT', value: 'jenkins/lettuce'),
string(name: 'CREATE_DEPLOYMENT', value: create_deployment), string(name: 'BUILD_STATUS', value: build_status)
],
propagate: false, wait: false
if (currentBuild.currentResult != "SUCCESS"){
slackSend "`${JOB_NAME}` #${BUILD_NUMBER}: ${currentBuild.currentResult} after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}"
email_body = "See: <${BUILD_URL}>\n\nChanges:\n"
change_sets = currentBuild.changeSets
for (int j = 0; j < change_sets.size(); j++) {
change_set_items = change_sets[j].items
for (int k = 0; k < change_set_items.length; k++) {
item = change_set_items[k]
email_body = email_body + "\n Commit: ${item.commitId} by ${item.author}: ${item.msg}"
}
}
emailext body: email_body,
subject: "Build failed in Jenkins: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org'
} else if (currentBuild.currentResult == "SUCCESS" && currentBuild.previousBuild.currentResult != "SUCCESS") {
slackSend "`${JOB_NAME}` #${BUILD_NUMBER}: Back to normal after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}"
emailext body: "See <${BUILD_URL}>",
subject: "Jenkins Build is back to normal: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org'
}
}
}
}
}
}

141
scripts/Jenkinsfiles/python
View File

@@ -1,10 +1,17 @@
def runPythonTests() {
// Determine git refspec, branch, and clone type
if (env.ghprbActualCommit) {
git_branch = "${ghprbActualCommit}"
git_refspec = "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
} else {
git_branch = 'master'
git_refspec = "+refs/heads/master:refs/remotes/origin/master"
}
sshagent(credentials: ['jenkins-worker', 'jenkins-worker-pem'], ignoreMissing: true) {
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${ghprbActualCommit}']],
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: git_branch]],
doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', honorRefspec: true,
noTags: true, shallow: true]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'jenkins-worker',
refspec: '+refs/heads/master:refs/remotes/origin/master +refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*',
url: 'git@github.com:edx/edx-platform.git']]]
refspec: git_refspec, url: 'git@github.com:edx/edx-platform.git']]]
console_output = sh(returnStdout: true, script: 'bash scripts/all-tests.sh').trim()
dir('stdout') {
writeFile file: "${TEST_SUITE}-stdout.log", text: console_output
@@ -33,6 +40,26 @@ pipeline {
XDIST_GIT_BRANCH = "${ghprbActualCommit}"
}
stages {
stage('Mark build as pending on Github') {
when {
// Only run github-build-status for master builds
expression { env.ghprbActualCommit == null }
}
steps {
script {
commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
build job: 'github-build-status',
parameters: [
string(name: 'GIT_SHA', value: commit_sha),
string(name: 'GITHUB_ORG', value: 'edx'), string(name: 'GITHUB_REPO', value: 'edx-platform'),
string(name: 'TARGET_URL', value: "${BUILD_URL}"),
string(name: 'DESCRIPTION', value: 'Pending'), string(name: 'CONTEXT', value: 'jenkins/python'),
string(name: 'CREATE_DEPLOYMENT', value: 'false'), string(name: 'BUILD_STATUS', value: 'pending')
],
propagate: false, wait: false
}
}
}
stage('Run Tests') {
parallel {
stage("lms-unit") {
@@ -100,32 +127,102 @@ pipeline {
stage('Run coverage') {
environment {
CODE_COV_TOKEN = credentials('CODE_COV_TOKEN')
TARGET_BRANCH = "origin/master"
CI_BRANCH = "${ghprbSourceBranch}"
SUBSET_JOB = "null" // Keep this variable until we can remove the $SUBSET_JOB path from .coveragerc
}
steps {
sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) {
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '${ghprbActualCommit}']],
doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption',
honorRefspec: true, noTags: true, shallow: true]], submoduleCfg: [],
userRemoteConfigs: [[credentialsId: 'jenkins-worker',
refspec: '+refs/heads/master:refs/remotes/origin/master +refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*',
url: 'git@github.com:edx/edx-platform.git']]]
unstash 'lms-unit-reports'
unstash 'cms-unit-reports'
unstash 'commonlib-unit-reports'
sh "./scripts/jenkins-report.sh"
script {
if (env.ghprbActualCommit) {
git_branch = "${ghprbActualCommit}"
git_refspec = "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
ci_branch = "${ghprbSourceBranch}"
target_branch = "origin/${ghprbTargetBranch}"
} else {
git_branch = 'master'
git_refspec = "+refs/heads/master:refs/remotes/origin/master"
ci_branch = "master"
target_branch = "origin/master"
}
sshagent(credentials: ['jenkins-worker'], ignoreMissing: true) {
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: git_branch]],
doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption',
honorRefspec: true, noTags: true, shallow: true]], submoduleCfg: [],
userRemoteConfigs: [[credentialsId: 'jenkins-worker',
refspec: git_refspec, url: 'git@github.com:edx/edx-platform.git']]]
unstash 'lms-unit-reports'
unstash 'cms-unit-reports'
unstash 'commonlib-unit-reports'
sh """export CI_BRANCH=$ci_branch
export TARGET_BRANCH=$target_branch
./scripts/jenkins-report.sh"""
}
}
}
post {
always {
publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true,
reportDir: 'reports', reportFiles: 'diff_coverage_combined.html',
reportName: 'Diff Coverage Report', reportTitles: ''])
publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true,
reportDir: 'reports/cover', reportFiles: 'index.html',
reportName: 'Coverage.py Report', reportTitles: ''])
script {
publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true,
reportDir: 'reports/cover', reportFiles: 'index.html',
reportName: 'Coverage.py Report', reportTitles: ''])
if (env.ghprbPullId != null) {
publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true,
reportDir: 'reports', reportFiles: 'diff_coverage_combined.html',
reportName: 'Diff Coverage Report', reportTitles: ''])
}
}
}
}
}
}
post {
always {
script{
if (env.ghprbPullId != null) {
// For PR jobs, run the edx-platform-test-notifier for PR reporting
build job: 'edx-platform-test-notifier', parameters: [string(name: 'PR_NUMBER', value: "${ghprbPullId}")], wait: false
} else {
// For master jobs run github-build-status and report to slack when necessary
if (currentBuild.currentResult == "SUCCESS") {
create_deployment = "true"
build_status = "success"
build_description = "Build Passed"
}
else {
create_deployment = "false"
build_status = "failure"
build_description = "Build Failed"
}
commit_sha = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
build job: 'github-build-status',
parameters: [
string(name: 'GIT_SHA', value: commit_sha),
string(name: 'GITHUB_ORG', value: 'edx'), string(name: 'GITHUB_REPO', value: 'edx-platform'),
string(name: 'TARGET_URL', value: "${BUILD_URL}"),
string(name: 'DESCRIPTION', value: build_description), string(name: 'CONTEXT', value: 'jenkins/python'),
string(name: 'CREATE_DEPLOYMENT', value: create_deployment), string(name: 'BUILD_STATUS', value: build_status)
],
propagate: false, wait: false
if (currentBuild.currentResult != "SUCCESS"){
slackSend "`${JOB_NAME}` #${BUILD_NUMBER}: ${currentBuild.currentResult} after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}"
email_body = "See: <${BUILD_URL}>\n\nChanges:\n"
change_sets = currentBuild.changeSets
for (int j = 0; j < change_sets.size(); j++) {
change_set_items = change_sets[j].items
for (int k = 0; k < change_set_items.length; k++) {
item = change_set_items[k]
email_body = email_body + "\n Commit: ${item.commitId} by ${item.author}: ${item.msg}"
}
}
emailext body: email_body,
subject: "Build failed in Jenkins: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org'
} else if (currentBuild.currentResult == "SUCCESS" && currentBuild.previousBuild.currentResult != "SUCCESS") {
slackSend "`${JOB_NAME}` #${BUILD_NUMBER}: Back to normal after ${currentBuild.durationString.replace(' and counting', '')}\n${BUILD_URL}"
emailext body: "See <${BUILD_URL}>",
subject: "Jenkins Build is back to normal: ${JOB_NAME} #${BUILD_NUMBER}", to: 'testeng@edx.org'
}
}
}
}