From e91cf37bf7e7b9af02da7c7b59dea01a8315fba7 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Wed, 26 Jun 2019 14:09:28 -0400 Subject: [PATCH 1/3] ignore PRs that aren't Open Sometimes it is necessary recreate the Jenkins PR polling job, or cleanup old runs. This can cause Jenkins to lose the record of which PRs it has already seen, and retest old PRs which are long closed. This change causes the polling job to check the state of the PR, and ignore it if it isn't Open. Testing PRs which are in any other state doesn't provide any value. --- .../templates/mbs-polling-pagure.yaml | 40 +++++++++++++++---- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml b/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml index 9e5b377f..04221b52 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml +++ b/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml @@ -90,6 +90,8 @@ objects: jenkinsPipelineStrategy: jenkinsfile: |- // Don't use external Jenkinsfile here, or Jenkins will also poll on that repo and branch + library identifier: 'c3i@master', changelog: false, + retriever: modernSCM([$class: 'GitSCMSource', remote: 'https://pagure.io/c3i-library.git']) pipeline { agent { kubernetes { @@ -131,10 +133,12 @@ objects: environment { PIPELINE_NAMESPACE = readFile('/run/secrets/kubernetes.io/serviceaccount/namespace').trim() PAGURE_URL = "${PAGURE_URL}" + PAGURE_API = "${env.PAGURE_URL}/api/0" + PAGURE_REPO_NAME = "${PAGURE_REPO_NAME}" PAGURE_REPO_IS_FORK = "${PAGURE_REPO_IS_FORK}" PAGURE_POLLING_FOR_PR = "${PAGURE_POLLING_FOR_PR}" - PAGURE_REPO_HOME = "${env.PAGURE_URL}${env.PAGURE_REPO_IS_FORK == 'true' ? '/fork' : ''}/${PAGURE_REPO_NAME}" - GIT_URL = "${env.PAGURE_URL}/${env.PAGURE_REPO_IS_FORK == 'true' ? 'forks/' : ''}${PAGURE_REPO_NAME}.git" + PAGURE_REPO_HOME = "${env.PAGURE_URL}${env.PAGURE_REPO_IS_FORK == 'true' ? '/fork' : ''}/${env.PAGURE_REPO_NAME}" + GIT_URL = "${env.PAGURE_URL}/${env.PAGURE_REPO_IS_FORK == 'true' ? 'forks/' : ''}${env.PAGURE_REPO_NAME}.git" PREMERGE_JOB_NAME = "${PREMERGE_JOB_NAME}" POSTMERGE_JOB_NAME = "${POSTMERGE_JOB_NAME}" } @@ -169,12 +173,20 @@ objects: currentBuild.description = """${currentBuild.displayName}""" } else if (env.PAGURE_POLLING_FOR_PR == 'true' && branch ==~ /^pull\/[0-9]+\/head$/) { env.PR_NO = branch.split('/')[1] - env.PR_URL = "${env.PAGURE_REPO_HOME}/pull-request/${env.PR_NO}" - // To HTML syntax in build description, go to `Jenkins/Global Security/Markup Formatter` and select 'Safe HTML'. - def pagureLink = """PR#${env.PR_NO}""" - echo "Building PR #${env.PR_NO}: ${env.PR_URL}" - currentBuild.displayName = "PR#${env.PR_NO}" - currentBuild.description = pagureLink + def prInfo = withPagure { + it.getPR(env.PR_NO) + } + if (prInfo.status == 'Open') { + env.PR_URL = "${env.PAGURE_REPO_HOME}/pull-request/${env.PR_NO}" + // To HTML syntax in build description, go to `Jenkins/Global Security/Markup Formatter` and select 'Safe HTML'. + def pagureLink = """PR#${env.PR_NO}""" + echo "Building PR #${env.PR_NO}: ${env.PR_URL}" + currentBuild.displayName = "PR#${env.PR_NO}" + currentBuild.description = pagureLink + } else { + echo "Skipping PR#${env.PR_NO} because it is ${prInfo.status}" + env.SKIP = 'true' + } } else { // This shouldn't happen. error("Build is aborted due to unexpected polling trigger actions.") } @@ -209,6 +221,11 @@ objects: } } stage('Build') { + when { + not { + environment name: 'SKIP', value: 'true' + } + } steps { script { openshift.withCluster() { @@ -231,3 +248,10 @@ objects: } } } + def withPagure(args=[:], cl) { + args.apiUrl = env.PAGURE_API + args.repo = env.PAGURE_REPO_NAME + args.isFork = env.PAGURE_REPO_IS_FORK == 'true' + def pagureClient = pagure.client(args) + return cl(pagureClient) + } From 30df0a82ca925525e99d7164fe6e18b9f1422651 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Wed, 26 Jun 2019 13:00:40 -0400 Subject: [PATCH 2/3] optimize checkouts Only checkout exactly the code we need to run the tests. This should help work around errors and timeouts (early EOF/index-pack failed) from Pagure. Jenkins seems to require a minimum of depth=2 or it fails when walking the revision history. Using a shallow clone is safe for PRs because we're always building from the head of the PR branch (OpenShift doesn't have a way to specify a non-head commit on a PR branch). When building from the master branch we use depth=10 to avoid a race condition in case multiple commits have been pushed in quick succession. The integration test job is launched using the exact revision of the code that was used to build the images, to ensure the results are valid. --- .../pipelines/templates/mbs-build.Jenkinsfile | 13 ++++++++++++- .../templates/mbs-integration-test.Jenkinsfile | 15 ++++++++++++++- .../pipelines/templates/mbs-polling-pagure.yaml | 15 +++++++++++---- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile b/openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile index 010ae8c1..c37e9b43 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile +++ b/openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile @@ -56,9 +56,20 @@ pipeline { steps { script { // check out specified branch/commit + def srcRef = env.MBS_GIT_REF.startsWith('pull/') ? env.MBS_GIT_REF : "heads/${env.MBS_GIT_REF}" checkout([$class: 'GitSCM', branches: [[name: params.MBS_GIT_REF]], - userRemoteConfigs: [[url: params.MBS_GIT_REPO, refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pull/*/head']], + userRemoteConfigs: [ + [ + name: 'origin', + url: params.MBS_GIT_REPO, + refspec: "+refs/${srcRef}:refs/remotes/origin/${env.MBS_GIT_REF}", + ], + ], + extensions: [ + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', noTags: true, shallow: true, depth: 2, honorRefspec: true], + ], ]) // get current commit ID diff --git a/openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile b/openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile index 27d8f2c4..3aa0d55e 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile +++ b/openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile @@ -55,10 +55,23 @@ pipeline { // Don't set ENVIRONMENT_LABEL in the environment block! Otherwise you will get 2 different UUIDs. env.ENVIRONMENT_LABEL = "test-${env.TEST_ID}" + def srcRef = params.MBS_GIT_REF.startsWith('pull/') ? params.MBS_GIT_REF : 'heads/master' + def localRef = params.MBS_GIT_REF.startsWith('pull/') ? params.MBS_GIT_REF : 'master' + def cloneDepth = params.MBS_GIT_REF.startsWith('pull/') ? 2 : 10 // check out specified branch/commit checkout([$class: 'GitSCM', branches: [[name: params.MBS_GIT_REF]], - userRemoteConfigs: [[url: params.MBS_GIT_REPO, refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pull/*/head']], + userRemoteConfigs: [ + [ + name: 'origin', + url: params.MBS_GIT_REPO, + refspec: "+refs/${srcRef}:refs/remotes/origin/${localRef}", + ], + ], + extensions: [ + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', noTags: true, shallow: true, depth: cloneDepth, honorRefspec: true], + ], ]) // get current commit ID diff --git a/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml b/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml index 04221b52..27137b58 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml +++ b/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml @@ -150,16 +150,20 @@ objects: script { // checking out the polled branch def polledBranch = env.PAGURE_POLLING_FOR_PR == 'true' ? 'origin/pull/*/head' : "origin/${PAGURE_POLLED_BRANCH}" + def srcRef = env.PAGURE_POLLING_FOR_PR == 'true' ? 'pull/*/head' : "heads/${PAGURE_POLLED_BRANCH}" def scmVars = checkout([$class: 'GitSCM', branches: [[name: polledBranch]], userRemoteConfigs: [ [ name: 'origin', url: env.GIT_URL, - refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pull/*/head', + refspec: "+refs/${srcRef}:refs/remotes/${polledBranch}", ], ], - extensions: [[$class: 'CleanBeforeCheckout']], + extensions: [ + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', noTags: true, shallow: true, depth: 2, honorRefspec: true], + ], ]) env.MBS_GIT_COMMIT = scmVars.GIT_COMMIT // setting build display name @@ -206,10 +210,13 @@ objects: [ name: 'origin', url: env.GIT_URL, - refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pull/*/head', + refspec: "+refs/heads/${env.MBS_GIT_BRANCH}:refs/remotes/origin/${env.MBS_GIT_BRANCH}", ], ], - extensions: [[$class: 'CleanBeforeCheckout']], + extensions: [ + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', noTags: true, shallow: true, depth: 2, honorRefspec: true], + ], ]) script { dir('openshift/integration/koji/pipelines') { From 934bc4bd050da4488f5dd8043b12999f6e945230 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Thu, 27 Jun 2019 11:36:13 -0400 Subject: [PATCH 3/3] wrap checkouts in retry(5) Avoid failing jobs due to temporary errors when checking out from Pagure. This should help work around transient DNS/network failures. --- .../pipelines/templates/mbs-build.Jenkinsfile | 28 ++++----- .../mbs-integration-test.Jenkinsfile | 30 +++++----- .../templates/mbs-polling-pagure.yaml | 57 ++++++++++--------- 3 files changed, 62 insertions(+), 53 deletions(-) diff --git a/openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile b/openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile index c37e9b43..de25fd41 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile +++ b/openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile @@ -57,20 +57,22 @@ pipeline { script { // check out specified branch/commit def srcRef = env.MBS_GIT_REF.startsWith('pull/') ? env.MBS_GIT_REF : "heads/${env.MBS_GIT_REF}" - checkout([$class: 'GitSCM', - branches: [[name: params.MBS_GIT_REF]], - userRemoteConfigs: [ - [ - name: 'origin', - url: params.MBS_GIT_REPO, - refspec: "+refs/${srcRef}:refs/remotes/origin/${env.MBS_GIT_REF}", + retry(5) { + checkout([$class: 'GitSCM', + branches: [[name: params.MBS_GIT_REF]], + userRemoteConfigs: [ + [ + name: 'origin', + url: params.MBS_GIT_REPO, + refspec: "+refs/${srcRef}:refs/remotes/origin/${env.MBS_GIT_REF}", + ], ], - ], - extensions: [ - [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', noTags: true, shallow: true, depth: 2, honorRefspec: true], - ], - ]) + extensions: [ + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', noTags: true, shallow: true, depth: 2, honorRefspec: true], + ], + ]) + } // get current commit ID // FIXME: Due to a bug discribed in https://issues.jenkins-ci.org/browse/JENKINS-45489, diff --git a/openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile b/openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile index 3aa0d55e..4728d8b9 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile +++ b/openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile @@ -58,21 +58,23 @@ pipeline { def srcRef = params.MBS_GIT_REF.startsWith('pull/') ? params.MBS_GIT_REF : 'heads/master' def localRef = params.MBS_GIT_REF.startsWith('pull/') ? params.MBS_GIT_REF : 'master' def cloneDepth = params.MBS_GIT_REF.startsWith('pull/') ? 2 : 10 - // check out specified branch/commit - checkout([$class: 'GitSCM', - branches: [[name: params.MBS_GIT_REF]], - userRemoteConfigs: [ - [ - name: 'origin', - url: params.MBS_GIT_REPO, - refspec: "+refs/${srcRef}:refs/remotes/origin/${localRef}", + retry(5) { + // check out specified branch/commit + checkout([$class: 'GitSCM', + branches: [[name: params.MBS_GIT_REF]], + userRemoteConfigs: [ + [ + name: 'origin', + url: params.MBS_GIT_REPO, + refspec: "+refs/${srcRef}:refs/remotes/origin/${localRef}", + ], ], - ], - extensions: [ - [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', noTags: true, shallow: true, depth: cloneDepth, honorRefspec: true], - ], - ]) + extensions: [ + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', noTags: true, shallow: true, depth: cloneDepth, honorRefspec: true], + ], + ]) + } // get current commit ID // FIXME: Due to a bug discribed in https://issues.jenkins-ci.org/browse/JENKINS-45489, diff --git a/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml b/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml index 27137b58..cd8d8177 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml +++ b/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml @@ -151,20 +151,23 @@ objects: // checking out the polled branch def polledBranch = env.PAGURE_POLLING_FOR_PR == 'true' ? 'origin/pull/*/head' : "origin/${PAGURE_POLLED_BRANCH}" def srcRef = env.PAGURE_POLLING_FOR_PR == 'true' ? 'pull/*/head' : "heads/${PAGURE_POLLED_BRANCH}" - def scmVars = checkout([$class: 'GitSCM', - branches: [[name: polledBranch]], - userRemoteConfigs: [ - [ - name: 'origin', - url: env.GIT_URL, - refspec: "+refs/${srcRef}:refs/remotes/${polledBranch}", + def scmVars + retry(5) { + scmVars = checkout([$class: 'GitSCM', + branches: [[name: polledBranch]], + userRemoteConfigs: [ + [ + name: 'origin', + url: env.GIT_URL, + refspec: "+refs/${srcRef}:refs/remotes/${polledBranch}", + ], ], - ], - extensions: [ - [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', noTags: true, shallow: true, depth: 2, honorRefspec: true], - ], - ]) + extensions: [ + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', noTags: true, shallow: true, depth: 2, honorRefspec: true], + ], + ]) + } env.MBS_GIT_COMMIT = scmVars.GIT_COMMIT // setting build display name def prefix = 'origin/' @@ -204,20 +207,22 @@ objects: } } steps { - checkout([$class: 'GitSCM', - branches: [[name: env.MBS_GIT_BRANCH]], - userRemoteConfigs: [ - [ - name: 'origin', - url: env.GIT_URL, - refspec: "+refs/heads/${env.MBS_GIT_BRANCH}:refs/remotes/origin/${env.MBS_GIT_BRANCH}", + retry(5) { + checkout([$class: 'GitSCM', + branches: [[name: env.MBS_GIT_BRANCH]], + userRemoteConfigs: [ + [ + name: 'origin', + url: env.GIT_URL, + refspec: "+refs/heads/${env.MBS_GIT_BRANCH}:refs/remotes/origin/${env.MBS_GIT_BRANCH}", + ], ], - ], - extensions: [ - [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', noTags: true, shallow: true, depth: 2, honorRefspec: true], - ], - ]) + extensions: [ + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', noTags: true, shallow: true, depth: 2, honorRefspec: true], + ], + ]) + } script { dir('openshift/integration/koji/pipelines') { sh '''