diff --git a/openshift/integration/koji/containers/jenkins-slave/Dockerfile b/openshift/integration/koji/containers/jenkins-slave/Dockerfile index ae8666f8..2c8167f8 100644 --- a/openshift/integration/koji/containers/jenkins-slave/Dockerfile +++ b/openshift/integration/koji/containers/jenkins-slave/Dockerfile @@ -32,7 +32,7 @@ USER root RUN ${DNF_CMD} install -y \ java-1.8.0-openjdk nss_wrapper gettext git \ - tar gzip skopeo wget make bind-utils \ + tar gzip skopeo wget make bind-utils python3-jinja2-cli \ origin-clients \ # Jenkins pipeline 'sh' steps seem to require ps procps-ng \ diff --git a/openshift/integration/koji/pipelines/Makefile b/openshift/integration/koji/pipelines/Makefile index de353e26..cfb8e06e 100644 --- a/openshift/integration/koji/pipelines/Makefile +++ b/openshift/integration/koji/pipelines/Makefile @@ -23,7 +23,7 @@ install: @for job in $(JOBS); do \ echo "[PIPELINE] Updating pipeline job \"$${job}\"..." ; \ template_file=$$(cat ./$(JOBS_DIR)/$${job}.tmpl); \ - $(OC_CMD) process --local -f ./$(TEMPLATES_DIR)/$${template_file} \ + jinja2 ./$(TEMPLATES_DIR)/$${template_file} | $(OC_CMD) process --local -f - \ --param-file ./$(JOBS_DIR)/$${job}.env | $(OC_CMD) apply -f -; \ echo "[PIPELINE] Pipeline job \"$${job}\" updated" ; \ done diff --git a/openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-prod.env b/openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-prod.env index e03e3abb..2bc83735 100644 --- a/openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-prod.env +++ b/openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-prod.env @@ -4,4 +4,6 @@ SUBJECT_IDENTIFIER_REGEX=^factory2/mbs-backend@sha256: SOURCE_CONTAINER_REPO=quay.io/factory2/mbs-backend TARGET_TAG=prod MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-backend-greenwave-promote-to-prod.VirtualTopic.eng.greenwave.decision.update -IMAGE_PROMOTION_JOB=mbs-backend-promoting-to-prod +PROMOTING_DESTINATIONS=quay.io/factory2/mbs-backend +TAG_INTO_IMAGESTREAM=true +DEST_IMAGESTREAM_NAME=mbs-backend diff --git a/openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-stage.env b/openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-stage.env index 2a007d11..e1de485f 100644 --- a/openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-stage.env +++ b/openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-stage.env @@ -4,4 +4,6 @@ SUBJECT_IDENTIFIER_REGEX=^factory2/mbs-backend@sha256: SOURCE_CONTAINER_REPO=quay.io/factory2/mbs-backend TARGET_TAG=stage MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-backend-greenwave-promote-to-stage.VirtualTopic.eng.greenwave.decision.update -IMAGE_PROMOTION_JOB=mbs-backend-promoting-to-stage +PROMOTING_DESTINATIONS=quay.io/factory2/mbs-backend +TAG_INTO_IMAGESTREAM=true +DEST_IMAGESTREAM_NAME=mbs-backend diff --git a/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-prod.env b/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-prod.env deleted file mode 100644 index 44ee43d6..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-prod.env +++ /dev/null @@ -1,5 +0,0 @@ -NAME=mbs-backend-promoting-to-prod -PROMOTING_DESTINATIONS=quay.io/factory2/mbs-backend -DEST_TAG=prod -TAG_INTO_IMAGESTREAM=true -DEST_IMAGESTREAM_NAME=mbs-backend diff --git a/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-prod.tmpl b/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-prod.tmpl deleted file mode 100644 index cd70fa7a..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-prod.tmpl +++ /dev/null @@ -1 +0,0 @@ -mbs-image-promotion-template.yaml diff --git a/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-stage.env b/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-stage.env deleted file mode 100644 index 9439ca67..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-stage.env +++ /dev/null @@ -1,5 +0,0 @@ -NAME=mbs-backend-promoting-to-stage -PROMOTING_DESTINATIONS=quay.io/factory2/mbs-backend -DEST_TAG=stage -TAG_INTO_IMAGESTREAM=true -DEST_IMAGESTREAM_NAME=mbs-backend diff --git a/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-stage.tmpl b/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-stage.tmpl deleted file mode 100644 index cd70fa7a..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-stage.tmpl +++ /dev/null @@ -1 +0,0 @@ -mbs-image-promotion-template.yaml diff --git a/openshift/integration/koji/pipelines/jobs/mbs-dev-integration-test.env b/openshift/integration/koji/pipelines/jobs/mbs-dev-integration-test.env deleted file mode 100644 index 199953f1..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-dev-integration-test.env +++ /dev/null @@ -1 +0,0 @@ -NAME=mbs-dev-integration-test diff --git a/openshift/integration/koji/pipelines/jobs/mbs-dev-integration-test.tmpl b/openshift/integration/koji/pipelines/jobs/mbs-dev-integration-test.tmpl deleted file mode 100644 index 467c9eea..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-dev-integration-test.tmpl +++ /dev/null @@ -1 +0,0 @@ -mbs-integration-test-template.yaml diff --git a/openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-prod.env b/openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-prod.env index 6d80b8fa..55af128e 100644 --- a/openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-prod.env +++ b/openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-prod.env @@ -4,4 +4,6 @@ SUBJECT_IDENTIFIER_REGEX=^factory2/mbs-frontend@sha256: SOURCE_CONTAINER_REPO=quay.io/factory2/mbs-frontend TARGET_TAG=prod MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-frontend-greenwave-promote-to-prod.VirtualTopic.eng.greenwave.decision.update -IMAGE_PROMOTION_JOB=mbs-frontend-promoting-to-prod +PROMOTING_DESTINATIONS=quay.io/factory2/mbs-frontend +TAG_INTO_IMAGESTREAM=true +DEST_IMAGESTREAM_NAME=mbs-frontend diff --git a/openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-stage.env b/openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-stage.env index a1be5810..80022fb8 100644 --- a/openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-stage.env +++ b/openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-stage.env @@ -4,4 +4,6 @@ SUBJECT_IDENTIFIER_REGEX=^factory2/mbs-frontend@sha256: SOURCE_CONTAINER_REPO=quay.io/factory2/mbs-frontend TARGET_TAG=stage MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-frontend-greenwave-promote-to-stage.VirtualTopic.eng.greenwave.decision.update -IMAGE_PROMOTION_JOB=mbs-frontend-promoting-to-stage +PROMOTING_DESTINATIONS=quay.io/factory2/mbs-frontend +TAG_INTO_IMAGESTREAM=true +DEST_IMAGESTREAM_NAME=mbs-frontend diff --git a/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-prod.env b/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-prod.env deleted file mode 100644 index 40271273..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-prod.env +++ /dev/null @@ -1,5 +0,0 @@ -NAME=mbs-frontend-promoting-to-prod -PROMOTING_DESTINATIONS=quay.io/factory2/mbs-frontend -DEST_TAG=prod -TAG_INTO_IMAGESTREAM=true -DEST_IMAGESTREAM_NAME=mbs-frontend diff --git a/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-prod.tmpl b/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-prod.tmpl deleted file mode 100644 index cd70fa7a..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-prod.tmpl +++ /dev/null @@ -1 +0,0 @@ -mbs-image-promotion-template.yaml diff --git a/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-stage.env b/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-stage.env deleted file mode 100644 index 2c3d0e4f..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-stage.env +++ /dev/null @@ -1,5 +0,0 @@ -NAME=mbs-frontend-promoting-to-stage -PROMOTING_DESTINATIONS=quay.io/factory2/mbs-frontend -DEST_TAG=stage -TAG_INTO_IMAGESTREAM=true -DEST_IMAGESTREAM_NAME=mbs-frontend diff --git a/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-stage.tmpl b/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-stage.tmpl deleted file mode 100644 index cd70fa7a..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-stage.tmpl +++ /dev/null @@ -1 +0,0 @@ -mbs-image-promotion-template.yaml diff --git a/openshift/integration/koji/pipelines/jobs/mbs-prod-integration-test.env b/openshift/integration/koji/pipelines/jobs/mbs-prod-integration-test.env deleted file mode 100644 index c9d34e1e..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-prod-integration-test.env +++ /dev/null @@ -1,2 +0,0 @@ -NAME=mbs-prod-integration-test -ENVIRONMENT=prod diff --git a/openshift/integration/koji/pipelines/jobs/mbs-prod-integration-test.tmpl b/openshift/integration/koji/pipelines/jobs/mbs-prod-integration-test.tmpl deleted file mode 100644 index 467c9eea..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-prod-integration-test.tmpl +++ /dev/null @@ -1 +0,0 @@ -mbs-integration-test-template.yaml diff --git a/openshift/integration/koji/pipelines/jobs/mbs-stage-integration-test.env b/openshift/integration/koji/pipelines/jobs/mbs-stage-integration-test.env deleted file mode 100644 index f94244f7..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-stage-integration-test.env +++ /dev/null @@ -1,2 +0,0 @@ -NAME=mbs-stage-integration-test -ENVIRONMENT=stage diff --git a/openshift/integration/koji/pipelines/jobs/mbs-stage-integration-test.tmpl b/openshift/integration/koji/pipelines/jobs/mbs-stage-integration-test.tmpl deleted file mode 100644 index 467c9eea..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-stage-integration-test.tmpl +++ /dev/null @@ -1 +0,0 @@ -mbs-integration-test-template.yaml diff --git a/openshift/integration/koji/pipelines/jobs/mbs-trigger-on-latest-tag.env b/openshift/integration/koji/pipelines/jobs/mbs-trigger-on-latest-tag.env index af4de5b2..c8dfba3b 100644 --- a/openshift/integration/koji/pipelines/jobs/mbs-trigger-on-latest-tag.env +++ b/openshift/integration/koji/pipelines/jobs/mbs-trigger-on-latest-tag.env @@ -1,4 +1,4 @@ NAME=mbs-trigger-on-latest-tag MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-trigger-on-latest-tag.VirtualTopic.eng.repotracker.container.tag.> -TEST_JOB_NAME=mbs-stage-integration-test +ENVIRONMENT=stage TRACKED_TAG=latest diff --git a/openshift/integration/koji/pipelines/jobs/mbs-trigger-on-stage-tag.env b/openshift/integration/koji/pipelines/jobs/mbs-trigger-on-stage-tag.env index 9d31b6ca..ec81502e 100644 --- a/openshift/integration/koji/pipelines/jobs/mbs-trigger-on-stage-tag.env +++ b/openshift/integration/koji/pipelines/jobs/mbs-trigger-on-stage-tag.env @@ -1,4 +1,4 @@ NAME=mbs-trigger-on-stage-tag MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-trigger-on-stage-tag.VirtualTopic.eng.repotracker.container.tag.> -TEST_JOB_NAME=mbs-prod-integration-test TRACKED_TAG=stage +ENVIRONMENT=prod diff --git a/openshift/integration/koji/pipelines/templates/mbs-build-template.yaml b/openshift/integration/koji/pipelines/templates/mbs-build-template.yaml index 57fca670..4b80f907 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-build-template.yaml +++ b/openshift/integration/koji/pipelines/templates/mbs-build-template.yaml @@ -90,13 +90,6 @@ parameters: - name: MBS_FRONTEND_IMAGESTREAM_NAMESPACE displayName: Namespace of ImageStream for MBS frontend images required: false -- name: MBS_INTEGRATION_TEST_BUILD_CONFIG_NAME - displayName: Name of BuildConfig for running integration tests - required: true - value: mbs-dev-integration-test -- name: MBS_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE - displayName: Namespace of BuildConfig for running integration tests - required: false - name: FORCE_PUBLISH_IMAGE displayName: Whether to push the resulting image regardless of the Git branch value: "false" @@ -153,10 +146,19 @@ parameters: displayName: The lifetime of the OpenShift project allocated by C3I-as-a-Service. required: true value: "120" +- name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE + displayName: The namespace where the Pipeline-as-a-Service project request BuildConfig has been defined + required: false + value: c3i - name: CLEANUP displayName: Cleanup objects after the pipeline is complete required: true value: "true" +- name: ENVIRONMENT + displayName: environment name (dev/stage/prod) + required: true + value: dev +{% include "snippets/c3i-library-parameters.yaml" %} labels: template: mbs-build objects: @@ -223,10 +225,6 @@ objects: spec: runPolicy: "Parallel" completionDeadlineSeconds: 1800 - source: - git: - uri: "${MBS_GIT_REPO}" - ref: "${MBS_GIT_REF}" strategy: type: JenkinsPipeline jenkinsPipelineStrategy: @@ -263,10 +261,6 @@ objects: value: "${MBS_FRONTEND_IMAGESTREAM_NAMESPACE}" - name: MBS_MAIN_BRANCH value: "${MBS_MAIN_BRANCH}" - - name: MBS_INTEGRATION_TEST_BUILD_CONFIG_NAME - value: "${MBS_INTEGRATION_TEST_BUILD_CONFIG_NAME}" - - name: MBS_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE - value: "${MBS_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE}" - name: PAGURE_REPO_NAME value: "${PAGURE_REPO_NAME}" - name: PAGURE_REPO_IS_FORK @@ -285,6 +279,8 @@ objects: value: "${EXTRA_RPMS}" - name: TESTCASES value: "${TESTCASES}" + - name: ENVIRONMENT + value: "${ENVIRONMENT}" - name: USE_C3IAAS value: "${USE_C3IAAS}" - name: C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE @@ -293,6 +289,9 @@ objects: value: "${C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAME}" - name: C3IAAS_LIFETIME value: "${C3IAAS_LIFETIME}" + - name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE + value: "${PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE}" - name: CLEANUP value: "${CLEANUP}" - jenkinsfilePath: openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile + jenkinsfile: | + {% filter indent(width=10) %}{% include "mbs-build.Jenkinsfile" %}{% endfilter %} diff --git a/openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile b/openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile index 00759c76..c6ea1ef1 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile +++ b/openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile @@ -1,43 +1,7 @@ -library identifier: 'c3i@master', changelog: false, - retriever: modernSCM([$class: 'GitSCMSource', remote: 'https://pagure.io/c3i-library.git']) +{% include "snippets/c3i-library.groovy" %} import static org.apache.commons.lang.StringEscapeUtils.escapeHtml; pipeline { - agent { - kubernetes { - cloud params.JENKINS_AGENT_CLOUD_NAME - label "jenkins-slave-${UUID.randomUUID().toString()}" - serviceAccount params.JENKINS_AGENT_SERVICE_ACCOUNT - defaultContainer 'jnlp' - yaml """ - apiVersion: v1 - kind: Pod - metadata: - labels: - app: "jenkins-${env.JOB_BASE_NAME.take(50)}" - factory2-pipeline-kind: "mbs-build-pipeline" - factory2-pipeline-build-number: "${env.BUILD_NUMBER}" - spec: - containers: - - name: jnlp - image: "${params.JENKINS_AGENT_IMAGE}" - imagePullPolicy: Always - tty: true - env: - - name: REGISTRY_CREDENTIALS - valueFrom: - secretKeyRef: - name: "${params.CONTAINER_REGISTRY_CREDENTIALS}" - key: ".dockerconfigjson" - resources: - requests: - memory: 512Mi - cpu: 300m - limits: - memory: 768Mi - cpu: 500m - """ - } - } + {% include "snippets/default-agent.groovy" %} options { timestamps() timeout(time: 120, unit: 'MINUTES') @@ -64,6 +28,7 @@ pipeline { env.GIT_COMMIT = env.MBS_GIT_COMMIT echo "Build ${params.MBS_GIT_REF}, commit=${env.MBS_GIT_COMMIT}" + env.IMAGE_IS_SCRATCH = (params.MBS_GIT_REF != params.MBS_MAIN_BRANCH) // Is the current branch a pull-request? If no, env.PR_NO will be empty. env.PR_NO = getPrNo(params.MBS_GIT_REF) @@ -102,6 +67,7 @@ pipeline { -e '/sitepackages/a setenv = PYTHONPATH={toxinidir}' \ tox.ini """ + {% include "snippets/get_paas_domain.groovy" %} } } } @@ -304,33 +270,7 @@ pipeline { } } } - stage('Run integration tests') { - steps { - script { - openshift.withCluster() { - openshift.withProject(params.MBS_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE) { - def build = c3i.buildAndWait(script: this, objs: "bc/${params.MBS_INTEGRATION_TEST_BUILD_CONFIG_NAME}", - '-e', "MBS_GIT_REPO=${params.MBS_GIT_REPO}", - '-e', "MBS_GIT_REF=${env.PR_NO ? params.MBS_GIT_REF : env.MBS_GIT_COMMIT}", - '-e', "MBS_BACKEND_IMAGE=${env.BACKEND_IMAGE_REF}", - '-e', "MBS_FRONTEND_IMAGE=${env.FRONTEND_IMAGE_REF}", - '-e', "TEST_IMAGES=${env.BACKEND_IMAGE_REF},${env.FRONTEND_IMAGE_REF}", - '-e', "IMAGE_IS_SCRATCH=${params.MBS_GIT_REF != params.MBS_MAIN_BRANCH}", - '-e', "TEST_NAMESPACE=${env.PIPELINE_ID}", - '-e', "TESTCASES='${params.TESTCASES}'", - '-e', "CLEANUP=${params.CLEANUP}" - ) - echo 'Integration tests PASSED' - } - } - } - } - post { - failure { - echo 'Integration tests FAILED' - } - } - } + {% include "snippets/mbs-integration-test.groovy" %} stage('Push images') { when { expression { @@ -340,10 +280,13 @@ pipeline { } steps { script { - if (env.REGISTRY_CREDENTIALS) { - dir ("${env.HOME}/.docker") { - writeFile file: 'config.json', text: env.REGISTRY_CREDENTIALS - } + if (params.CONTAINER_REGISTRY_CREDENTIALS) { + dir ("${env.HOME}/.docker") { + openshift.withCluster() { + def dockerconf = openshift.selector('secret', params.CONTAINER_REGISTRY_CREDENTIALS).object().data['.dockerconfigjson'] + writeFile file: 'config.json', text: dockerconf, encoding: "Base64" + } + } } def registryToken = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/token') def copyDown = { name, src -> @@ -412,13 +355,13 @@ pipeline { steps { script { openshift.withCluster() { - openshift.withProject(params.MBS_BACKEND_IMAGESTREAM_NAMESPACE) { + openshift.withProject(params.MBS_BACKEND_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_ID) { def sourceRef = "${params.MBS_BACKEND_IMAGESTREAM_NAME}@${env.BACKEND_IMAGE_DIGEST}" def destRef = "${params.MBS_BACKEND_IMAGESTREAM_NAME}:${params.MBS_DEV_IMAGE_TAG}" echo "Tagging ${sourceRef} as ${destRef}..." openshift.tag(sourceRef, destRef) } - openshift.withProject(params.MBS_FRONTEND_IMAGESTREAM_NAMESPACE) { + openshift.withProject(params.MBS_FRONTEND_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_ID) { def sourceRef = "${params.MBS_FRONTEND_IMAGESTREAM_NAME}@${env.FRONTEND_IMAGE_DIGEST}" def destRef = "${params.MBS_FRONTEND_IMAGESTREAM_NAME}:${params.MBS_DEV_IMAGE_TAG}" echo "Tagging ${sourceRef} as ${destRef}..." @@ -442,13 +385,13 @@ pipeline { openshift.withProject(env.PIPELINE_ID) { if (env.BACKEND_IMAGE_TAG) { echo "Removing tag ${env.BACKEND_IMAGE_TAG} from the ${params.MBS_BACKEND_IMAGESTREAM_NAME} ImageStream..." - openshift.withProject(params.MBS_BACKEND_IMAGESTREAM_NAMESPACE) { + openshift.withProject(params.MBS_BACKEND_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_ID) { openshift.tag("${params.MBS_BACKEND_IMAGESTREAM_NAME}:${env.BACKEND_IMAGE_TAG}", "-d") } } if (env.FRONTEND_IMAGE_TAG) { echo "Removing tag ${env.FRONTEND_IMAGE_TAG} from the ${params.MBS_FRONTEND_IMAGESTREAM_NAME} ImageStream..." - openshift.withProject(params.MBS_FRONTEND_IMAGESTREAM_NAMESPACE) { + openshift.withProject(params.BS_FRONTEND_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_ID) { openshift.tag("${params.MBS_FRONTEND_IMAGESTREAM_NAME}:${env.FRONTEND_IMAGE_TAG}", "-d") } } @@ -539,3 +482,4 @@ def sendBuildStatusEmail(String status) { } emailext to: recipient, subject: subject, body: body } +{% include "snippets/functions.groovy" %} diff --git a/openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.Jenkinsfile b/openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.Jenkinsfile index 2add6d30..5cf996d5 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.Jenkinsfile +++ b/openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.Jenkinsfile @@ -1,79 +1,122 @@ -// Use scripted syntax because CIBuildTrigger currently doesn't support the declarative syntax -properties([ - buildDiscarder(logRotator(numToKeepStr: '10')), - pipelineTriggers([ - // example: https://github.com/jenkinsci/jms-messaging-plugin/blob/9b9387c3a52f037ba0d019c2ebcf2a2796fc6397/src/test/java/com/redhat/jenkins/plugins/ci/integration/AmqMessagingPluginIntegrationTest.java - [$class: 'CIBuildTrigger', - providerData: [$class: 'ActiveMQSubscriberProviderData', - name: params.MESSAGING_PROVIDER, - overrides: [topic: params.MESSAGING_TOPIC], - checks: [ - [field: '$.msg.subject_type', expectedValue: 'container-image'], - [field: '$.msg.subject_identifier', expectedValue: params.SUBJECT_IDENTIFIER_REGEX], - [field: '$.msg.decision_context', expectedValue: params.DECISION_CONTEXT_REGEX], - [field: '$.msg.policies_satisfied', expectedValue: 'true'], - ], - ], - ], - ]), -]) - -if (!params.CI_MESSAGE) { - echo 'This build is not started by a CI message. Only configurations were done.' - return -} - -def label = "jenkins-slave-${UUID.randomUUID().toString()}" -podTemplate( - cloud: "${params.JENKINS_AGENT_CLOUD_NAME}", - label: label, - serviceAccount: "${params.JENKINS_AGENT_SERVICE_ACCOUNT}", - defaultContainer: 'jnlp', - yaml: """ - apiVersion: v1 - kind: Pod - metadata: - labels: - app: "jenkins-${env.JOB_BASE_NAME.take(50)}" - factory2-pipeline-kind: "mbs-greenwave-trigger" - factory2-pipeline-build-number: "${env.BUILD_NUMBER}" - spec: - containers: - - name: jnlp - image: ${params.JENKINS_AGENT_IMAGE} - imagePullPolicy: Always - tty: true - resources: - requests: - memory: 256Mi - cpu: 200m - limits: - memory: 512Mi - cpu: 300m - """ -) { - node(label) { - stage('Trigger promotion') { - def message = readJSON text: params.CI_MESSAGE - // Extract the digest of the image to be promoted. - // e.g. factory2/waiverdb@sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b - // -> sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b - def digest = (message.msg.subject_identifier =~ /@(sha256:\w+)$/)[0][1] - // Generate the pull spec of the image - // e.g. quay.io/factory2/waiverdb@sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b - def image = "${params.SOURCE_CONTAINER_REPO}@${digest}" - echo "Starting a new build to promote image ${image} to :${params.TARGET_TAG}..." - openshift.withCluster() { - def bcSelector = openshift.selector('bc', params.IMAGE_PROMOTION_JOB) - def buildSelector = bcSelector.startBuild( - '-e', "IMAGE=${image}", - '-e', "DEST_TAG=${params.TARGET_TAG}", - ) - bcSelector.watch { - return !(it.object().status.phase in ["New", "Pending"]) +{% include "snippets/c3i-library.groovy" %} +pipeline { + {% include "snippets/default-agent.groovy" %} + options { + timestamps() + timeout(time: 30, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '10')) + } + environment { + PIPELINE_NAMESPACE = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/namespace').trim() + SERVICE_ACCOUNT_TOKEN = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/token').trim() + } + triggers { + ciBuildTrigger( + noSquash: false, + providerList: [ + activeMQSubscriber( + name: params.MESSAGING_PROVIDER, + overrides: [topic: params.MESSAGING_TOPIC], + checks: [ + [field: '$.msg.subject_type', expectedValue: 'container-image'], + [field: '$.msg.subject_identifier', expectedValue: params.SUBJECT_IDENTIFIER_REGEX], + [field: '$.msg.decision_context', expectedValue: params.DECISION_CONTEXT_REGEX], + [field: '$.msg.policies_satisfied', expectedValue: 'true'], + ] + ) + ] + ) + } + stages { + stage("Message Check and setup") { + steps { + script { + if (!params.CI_MESSAGE) { + error("This build is not started by a CI message. Only configurations were done.") } - buildInfo = buildSelector.object() - echo "Build ${buildInfo.metadata.annotations['openshift.io/jenkins-build-uri'] ?: buildInfo.metadata.name} started." + def message = readJSON text: params.CI_MESSAGE + // Extract the digest of the image to be promoted. + // e.g. factory2/waiverdb@sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b + // -> sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b + def digest = (message.msg.subject_identifier =~ /@(sha256:\w+)$/)[0][1] + // Generate the pull spec of the image + // e.g. quay.io/mkovarik/waiverdb@sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b + env.IMAGE = "${params.SOURCE_CONTAINER_REPO}@${digest}" + echo "Starting promotion of image ${env.IMAGE} to :${params.TARGET_TAG}..." + // Setting up registry credentials + dir ("${env.HOME}/.docker") { + // for the OpenShift internal registry + def dockerConfig = readJSON text: '{ "auths": {} }' + dockerConfig.auths['docker-registry.default.svc:5000'] = [ + 'email': '', + 'auth': sh(returnStdout: true, script: 'set +x; echo -n "serviceaccount:$SERVICE_ACCOUNT_TOKEN" | base64 -').trim() + ] + // merging user specified credentials + if (params.CONTAINER_REGISTRY_CREDENTIALS) { + openshift.withCluster() { + def dockerconf = openshift.selector('secret', params.CONTAINER_REGISTRY_CREDENTIALS).object().data['.dockerconfigjson'] + def dockerString = new String(dockerconf.decodeBase64()) + toBeMerged = readJSON text: dockerString + dockerConfig.auths.putAll(toBeMerged.auths) + } + } + // writing to ~/.docker/config.json + writeJSON file: 'config.json', json: dockerConfig + } + } + } + } + stage('Pull image') { + steps { + echo "Pulling container image ${env.IMAGE}..." + withEnv(["SOURCE_IMAGE_REF=${env.IMAGE}"]) { + sh ''' + set -e +x # hide the token from Jenkins console + mkdir -p _image + skopeo copy docker://"$SOURCE_IMAGE_REF" dir:_image + ''' + } + } + } + stage('Promote image') { + steps { + script { + def destinations = params.PROMOTING_DESTINATIONS ? params.PROMOTING_DESTINATIONS.split(',') : [] + openshift.withCluster() { + def pushTasks = destinations.collectEntries { + ["Pushing ${it}" : { + def dest = "${it}:${params.TARGET_TAG}" + // Only docker and atomic registries are allowed + if (!dest.startsWith('atomic:') && !dest.startsWith('docker://')) { + dest = "docker://${dest}" + } + echo "Pushing container image to ${dest}..." + withEnv(["DEST_IMAGE_REF=${dest}"]) { + retry(5) { + sh 'skopeo copy dir:_image "$DEST_IMAGE_REF"' + } + } + }] + } + parallel pushTasks + } + } + } + } + stage('Tag ImageStream') { + when { + expression { + return params.DEST_IMAGESTREAM_NAME && params.TAG_INTO_IMAGESTREAM == "true" + } + } + steps { + script { + def destRef = "${params.DEST_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_NAMESPACE}/${params.DEST_IMAGESTREAM_NAME}:${params.TARGET_TAG}" + openshift.withCluster() { + echo "Tagging ${env.IMAGE} into ${destRef}..." + openshift.tag('--source=docker', env.IMAGE, destRef) + } + } } } } diff --git a/openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.yaml b/openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.yaml index 0d36c31f..8fc17e8f 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.yaml +++ b/openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.yaml @@ -32,9 +32,18 @@ parameters: - name: TARGET_TAG displayName: Tag name to promote the image to required: true -- name: IMAGE_PROMOTION_JOB - displayName: Downstream image promotion job to trigger +- name: TAG_INTO_IMAGESTREAM + displayName: Whether to tag the image into an ImageStream + value: "false" required: true +- name: DEST_IMAGESTREAM_NAME + displayName: Name of the ImageStream to be tagged + required: false + value: "" +- name: CONTAINER_REGISTRY_CREDENTIALS + displayName: Secret name of container registries used for pulling and pushing images + value: factory2-pipeline-registry-credentials + required: false - name: MESSAGING_PROVIDER displayName: Name of the JMS messaging provider value: Red Hat UMB @@ -47,6 +56,11 @@ parameters: - name: JENKINS_AGENT_CLOUD_NAME displayName: Name of OpenShift cloud in Jenkins master configuration value: openshift +- name: PROMOTING_DESTINATIONS + displayName: Comma seperated list of container repositories (without tags) to which the image will be promoted + description: OpenShift registries must be prefixed with 'atomic:' + required: true +{% include "snippets/c3i-library-parameters.yaml" %} objects: - kind: ServiceAccount apiVersion: v1 @@ -74,14 +88,12 @@ objects: spec: runPolicy: "Parallel" completionDeadlineSeconds: 1800 - source: - git: - uri: "${MBS_GIT_REPO}" - ref: "${MBS_GIT_REF}" strategy: type: JenkinsPipeline jenkinsPipelineStrategy: env: + - name: PROMOTING_DESTINATIONS + value: "${PROMOTING_DESTINATIONS}" - name: JENKINS_AGENT_CLOUD_NAME value: "${JENKINS_AGENT_CLOUD_NAME}" - name: JENKINS_AGENT_IMAGE @@ -90,10 +102,14 @@ objects: value: "${NAME}-jenkins-slave" - name: SOURCE_CONTAINER_REPO value: "${SOURCE_CONTAINER_REPO}" + - name: CONTAINER_REGISTRY_CREDENTIALS + value: "${CONTAINER_REGISTRY_CREDENTIALS}" - name: TARGET_TAG value: "${TARGET_TAG}" - - name: IMAGE_PROMOTION_JOB - value: "${IMAGE_PROMOTION_JOB}" + - name: TAG_INTO_IMAGESTREAM + value: "${TAG_INTO_IMAGESTREAM}" + - name: DEST_IMAGESTREAM_NAME + value: "${DEST_IMAGESTREAM_NAME}" - name: DECISION_CONTEXT_REGEX value: "${DECISION_CONTEXT_REGEX}" - name: SUBJECT_IDENTIFIER_REGEX @@ -107,4 +123,5 @@ objects: value: - name: MESSAGE_HEADERS value: - jenkinsfilePath: openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.Jenkinsfile + jenkinsfile: | + {% filter indent(width=10) %}{% include "mbs-greenwave-trigger.Jenkinsfile" %}{% endfilter %} diff --git a/openshift/integration/koji/pipelines/templates/mbs-image-promotion-template.yaml b/openshift/integration/koji/pipelines/templates/mbs-image-promotion-template.yaml deleted file mode 100644 index 4a38ab5e..00000000 --- a/openshift/integration/koji/pipelines/templates/mbs-image-promotion-template.yaml +++ /dev/null @@ -1,120 +0,0 @@ -# Template to produce a pipeline for promoting images between environments -# -# The pipeline pulls the image to be promoted, then pushes it to destinations with promoted tags. -# Optionally, it tags the promoted image into an image stream after pushes. ---- -apiVersion: v1 -kind: Template -metadata: - name: mbs-image-promotion -labels: - template: mbs-image-promotion -parameters: -- name: NAME - displayName: Short unique identifier for the templated instances - description: This field is used to deploy multiple pipelines to one OpenShift project from this template. - required: true -- name: MBS_GIT_REPO - displayName: MBS Git repo URL - description: Default MBS Git repo URL in which to run functional tests against - required: true - value: "https://pagure.io/fm-orchestrator.git" -- name: MBS_GIT_REF - displayName: MBS Git repo ref - description: Default MBS Git repo ref in which to run functional tests against - required: true - value: master -- name: IMAGE - displayName: The container image to be promoted - description: This field must be in repo:tag or repo@sha256 format - value: provided-by-trigger -- name: PROMOTING_DESTINATIONS - displayName: Comma seperated list of container repositories (without tags) to which the image will be promoted - description: OpenShift registries must be prefixed with 'atomic:' - required: true -- name: CONTAINER_REGISTRY_CREDENTIALS - displayName: Secret name of container registries used for pulling and pushing images - value: factory2-pipeline-registry-credentials - required: false -- name: TAG_INTO_IMAGESTREAM - displayName: Whether to tag the image into an ImageStream - value: "false" - required: true -- name: DEST_IMAGESTREAM_NAME - displayName: Name of the ImageStream to be tagged - required: false - value: "" -- name: DEST_IMAGESTREAM_NAMESPACE - displayName: Namespace of the ImageStream to be tagged - description: Leaving blank means using the same namespace as the pipeline build - required: false - value: "" -- name: DEST_TAG - displayName: Name of the new tag - required: true -- name: JENKINS_AGENT_IMAGE - displayName: Container image for Jenkins slave pods - required: true - value: quay.io/factory2/mbs-jenkins-slave:latest -- name: JENKINS_AGENT_CLOUD_NAME - displayName: Name of OpenShift cloud in Jenkins master configuration - required: true - value: openshift -objects: -- kind: ServiceAccount - apiVersion: v1 - metadata: - name: "${NAME}-jenkins-slave" - labels: - app: "${NAME}" -- kind: RoleBinding - apiVersion: v1 - metadata: - name: "${NAME}-jenkins-slave_edit" - labels: - app: "${NAME}" - subjects: - - kind: ServiceAccount - name: "${NAME}-jenkins-slave" - roleRef: - name: edit -- kind: "BuildConfig" - apiVersion: "v1" - metadata: - name: "${NAME}" - labels: - app: "${NAME}" - spec: - runPolicy: "Parallel" - completionDeadlineSeconds: 1800 - source: - git: - uri: "${MBS_GIT_REPO}" - ref: "${MBS_GIT_REF}" - strategy: - type: JenkinsPipeline - source: - type: None - jenkinsPipelineStrategy: - env: - - name: IMAGE - value: "${IMAGE}" - - name: PROMOTING_DESTINATIONS - value: "${PROMOTING_DESTINATIONS}" - - name: CONTAINER_REGISTRY_CREDENTIALS - value: "${CONTAINER_REGISTRY_CREDENTIALS}" - - name: TAG_INTO_IMAGESTREAM - value: "${TAG_INTO_IMAGESTREAM}" - - name: DEST_IMAGESTREAM_NAME - value: "${DEST_IMAGESTREAM_NAME}" - - name: DEST_IMAGESTREAM_NAMESPACE - value: "${DEST_IMAGESTREAM_NAMESPACE}" - - name: DEST_TAG - value: "${DEST_TAG}" - - name: JENKINS_AGENT_IMAGE - value: "${JENKINS_AGENT_IMAGE}" - - name: JENKINS_AGENT_CLOUD_NAME - value: "${JENKINS_AGENT_CLOUD_NAME}" - - name: JENKINS_AGENT_SERVICE_ACCOUNT - value: "${NAME}-jenkins-slave" - jenkinsfilePath: openshift/integration/koji/pipelines/templates/mbs-image-promotion.Jenkinsfile diff --git a/openshift/integration/koji/pipelines/templates/mbs-image-promotion.Jenkinsfile b/openshift/integration/koji/pipelines/templates/mbs-image-promotion.Jenkinsfile deleted file mode 100644 index c4e37857..00000000 --- a/openshift/integration/koji/pipelines/templates/mbs-image-promotion.Jenkinsfile +++ /dev/null @@ -1,125 +0,0 @@ -pipeline { - agent { - kubernetes { - cloud "${params.JENKINS_AGENT_CLOUD_NAME}" - label "jenkins-slave-${UUID.randomUUID().toString()}" - serviceAccount "${params.JENKINS_AGENT_SERVICE_ACCOUNT}" - defaultContainer 'jnlp' - yaml """ - apiVersion: v1 - kind: Pod - metadata: - labels: - app: "jenkins-${env.JOB_BASE_NAME.take(50)}" - factory2-pipeline-kind: "mbs-image-promotion-pipeline" - factory2-pipeline-build-number: "${env.BUILD_NUMBER}" - spec: - containers: - - name: jnlp - image: "${params.JENKINS_AGENT_IMAGE}" - imagePullPolicy: Always - tty: true - env: - - name: REGISTRY_CREDENTIALS - valueFrom: - secretKeyRef: - name: "${params.CONTAINER_REGISTRY_CREDENTIALS}" - key: '.dockerconfigjson' - resources: - requests: - memory: 512Mi - cpu: 300m - limits: - memory: 768Mi - cpu: 500m - """ - } - } - options { - timestamps() - timeout(time: 30, unit: 'MINUTES') - buildDiscarder(logRotator(numToKeepStr: '10')) - skipDefaultCheckout() - } - environment { - PIPELINE_NAMESPACE = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/namespace').trim() - SERVICE_ACCOUNT_TOKEN = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/token').trim() - } - stages { - stage ('Prepare') { - steps { - script { - // Setting up registry credentials - dir ("${env.HOME}/.docker") { - // for the OpenShift internal registry - def dockerConfig = readJSON text: '{ "auths": {} }' - dockerConfig.auths['docker-registry.default.svc:5000'] = [ - 'email': '', - 'auth': sh(returnStdout: true, script: 'set +x; echo -n "serviceaccount:$SERVICE_ACCOUNT_TOKEN" | base64 -').trim() - ] - // merging user specified credentials - if (env.REGISTRY_CREDENTIALS) { - toBeMerged = readJSON text: env.REGISTRY_CREDENTIALS - dockerConfig.auths.putAll(toBeMerged.auths) - } - // writing to ~/.docker/config.json - writeJSON file: 'config.json', json: dockerConfig - } - } - } - } - stage('Pull image') { - steps { - echo "Pulling container image ${params.IMAGE}..." - withEnv(["SOURCE_IMAGE_REF=${params.IMAGE}"]) { - sh ''' - set -e +x # hide the token from Jenkins console - mkdir -p _image - skopeo copy docker://"$SOURCE_IMAGE_REF" dir:_image - ''' - } - } - } - stage('Promote image') { - steps { - script { - def destinations = params.PROMOTING_DESTINATIONS ? params.PROMOTING_DESTINATIONS.split(',') : [] - openshift.withCluster() { - def pushTasks = destinations.collectEntries { - ["Pushing ${it}" : { - def dest = "${it}:${params.DEST_TAG}" - // Only docker and atomic registries are allowed - if (!dest.startsWith('atomic:') && !dest.startsWith('docker://')) { - dest = "docker://${dest}" - } - echo "Pushing container image to ${dest}..." - withEnv(["DEST_IMAGE_REF=${dest}"]) { - retry(5) { - sh 'skopeo copy dir:_image "$DEST_IMAGE_REF"' - } - } - }] - } - parallel pushTasks - } - } - } - } - stage('Tag ImageStream') { - when { - expression { - return params.DEST_IMAGESTREAM_NAME && params.TAG_INTO_IMAGESTREAM == "true" - } - } - steps { - script { - def destRef = "${params.DEST_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_NAMESPACE}/${params.DEST_IMAGESTREAM_NAME}:${params.DEST_TAG}" - openshift.withCluster() { - echo "Tagging ${params.IMAGE} into ${destRef}..." - openshift.tag('--source=docker', params.IMAGE, destRef) - } - } - } - } - } -} diff --git a/openshift/integration/koji/pipelines/templates/mbs-integration-test-template.yaml b/openshift/integration/koji/pipelines/templates/mbs-integration-test-template.yaml deleted file mode 100644 index 337554e2..00000000 --- a/openshift/integration/koji/pipelines/templates/mbs-integration-test-template.yaml +++ /dev/null @@ -1,143 +0,0 @@ -# Template to produce a new OpenShift pipeline for running integration tests -# ---- -apiVersion: v1 -kind: Template -metadata: - name: mbs-integration-test -labels: - template: mbs-integration-test -parameters: -- name: NAME - displayName: Short unique identifier for the templated instances - description: This field is used to deploy multiple pipelines to one OpenShift project from this template. - required: true - value: mbs-integration-test -- name: MBS_BACKEND_IMAGE - displayName: The MBS backend container image to be tested - description: This field must be in repo:tag or repo@sha256 format - value: quay.io/factory2/mbs-backend:latest -- name: MBS_FRONTEND_IMAGE - displayName: The MBS frontend container image to be tested - description: This field must be in repo:tag or repo@sha256 format - value: quay.io/factory2/mbs-frontend:latest -- name: MBS_GIT_REPO - displayName: MBS Git repo URL - description: Default MBS Git repo URL in which to find the integration tests to run - required: true - value: "https://pagure.io/fm-orchestrator.git" -- name: MBS_GIT_REF - displayName: MBS Git repo ref - description: Default MBS Git repo ref in which to find the integration tests to run - required: true - value: master -- name: KOJI_IMAGE - displayName: The Koji container image to be tested - description: This field must be in repo:tag or repo@sha256 format - value: quay.io/factory2/koji:latest -- name: TEST_IMAGES - displayName: Images being tested - description: >- - A space-separated list of the refs of the images being tested. - Results of the tests will be reported to ResultsDB. -- name: JENKINS_AGENT_IMAGE - displayName: Container image for Jenkins slave pods - required: true - value: quay.io/factory2/mbs-jenkins-slave:latest -- name: JENKINS_AGENT_CLOUD_NAME - displayName: Name of OpenShift cloud in Jenkins master configuration - required: true - value: openshift -- name: TEST_NAMESPACE - displayName: >- - Namespace where the Jenkins agent for this test will run, and where - test resources will be created. - required: false -- name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE - displayName: Namespace with pipeline-as-a-service build - value: c3i -- name: ENVIRONMENT - displayName: environment name (dev/stage/prod) - required: true - value: dev -- name: MESSAGING_PROVIDER - displayName: Name of the JMS messaging provider - value: Red Hat UMB -- name: TESTCASES - displayName: >- - Space-separated list of testcases to run as part of the pipeline. An empty string (the default) - causes all available testcases to run. The value "skip" causes no testcases to be run. - required: false - value: "" -- name: CLEANUP - displayName: Cleanup objects after testing is complete - required: true - value: "true" -objects: -- kind: ServiceAccount - apiVersion: v1 - metadata: - name: "${NAME}-jenkins-slave" - labels: - app: "${NAME}" -- kind: RoleBinding - apiVersion: v1 - metadata: - name: "${NAME}-jenkins-slave_edit" - labels: - app: "${NAME}" - subjects: - - kind: ServiceAccount - name: "${NAME}-jenkins-slave" - roleRef: - name: edit -- kind: "BuildConfig" - apiVersion: "v1" - metadata: - name: "${NAME}" - labels: - app: "${NAME}" - spec: - runPolicy: "Parallel" - completionDeadlineSeconds: 1800 - source: - git: - uri: "${MBS_GIT_REPO}" - ref: "${MBS_GIT_REF}" - strategy: - type: JenkinsPipeline - jenkinsPipelineStrategy: - env: - - name: MBS_GIT_REPO - value: "${MBS_GIT_REPO}" - - name: MBS_GIT_REF - value: "${MBS_GIT_REF}" - - name: MBS_BACKEND_IMAGE - value: "${MBS_BACKEND_IMAGE}" - - name: MBS_FRONTEND_IMAGE - value: "${MBS_FRONTEND_IMAGE}" - - name: KOJI_IMAGE - value: "${KOJI_IMAGE}" - - name: TEST_IMAGES - value: "${TEST_IMAGES}" - - name: IMAGE_IS_SCRATCH - value: "true" - - name: JENKINS_AGENT_IMAGE - value: "${JENKINS_AGENT_IMAGE}" - - name: JENKINS_AGENT_CLOUD_NAME - value: "${JENKINS_AGENT_CLOUD_NAME}" - - name: TEST_NAMESPACE - value: "${TEST_NAMESPACE}" - - name: ENVIRONMENT - value: "${ENVIRONMENT}" - - name: MESSAGING_PROVIDER - value: "${MESSAGING_PROVIDER}" - - name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE - value: "${PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE}" - - name: JENKINS_AGENT_SERVICE_ACCOUNT - value: "${NAME}-jenkins-slave" - - name: TESTCASES - value: "${TESTCASES}" - - name: CLEANUP - value: "${CLEANUP}" - jenkinsfilePath: openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile diff --git a/openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile b/openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile deleted file mode 100644 index 5ba423ea..00000000 --- a/openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile +++ /dev/null @@ -1,267 +0,0 @@ -library identifier: 'c3i@master', changelog: false, - retriever: modernSCM([$class: 'GitSCMSource', remote: 'https://pagure.io/c3i-library.git']) -def deployments -pipeline { - agent { - kubernetes { - cloud params.JENKINS_AGENT_CLOUD_NAME - label "jenkins-slave-${UUID.randomUUID().toString()}" - serviceAccount params.JENKINS_AGENT_SERVICE_ACCOUNT - defaultContainer 'jnlp' - yaml """ - apiVersion: v1 - kind: Pod - metadata: - labels: - app: "jenkins-${env.JOB_BASE_NAME.take(50)}" - factory2-pipeline-kind: "mbs-integration-test-pipeline" - factory2-pipeline-build-number: "${env.BUILD_NUMBER}" - spec: - containers: - - name: jnlp - image: "${params.JENKINS_AGENT_IMAGE}" - imagePullPolicy: Always - tty: true - resources: - requests: - memory: 512Mi - cpu: 300m - limits: - memory: 768Mi - cpu: 500m - """ - } - } - options { - timestamps() - timeout(time: 60, unit: 'MINUTES') - buildDiscarder(logRotator(numToKeepStr: '10')) - skipDefaultCheckout() - } - environment { - PIPELINE_ID = "${params.TEST_NAMESPACE}" - } - stages { - stage('Prepare') { - steps { - script { - // MBS_GIT_REF can be either a regular branch (in the heads/ namespace), a pull request - // branch (in the pull/ namespace), or a full 40-character sha1, which is assumed to - // exist on the master branch. - def branch = params.MBS_GIT_REF ==~ '[0-9a-f]{40}' ? 'master' : params.MBS_GIT_REF - c3i.clone(repo: params.MBS_GIT_REPO, branch: branch, rev: params.MBS_GIT_REF) - // get current commit ID - // FIXME: Due to a bug discribed in https://issues.jenkins-ci.org/browse/JENKINS-45489, - // the return value of checkout() is unreliable. - // Not working: env.MBS_GIT_COMMIT = scmVars.GIT_COMMIT - env.MBS_GIT_COMMIT = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() - echo "Running integration tests for ${branch}, commit=${env.MBS_GIT_COMMIT}" - currentBuild.displayName = "${branch}: ${env.MBS_GIT_COMMIT.take(7)}" - } - } - } - stage('Cleanup') { - when { - expression { - // Only run cleanup if we're running tests in the default namespace. - return !params.TEST_NAMESPACE - } - } - steps { - script { - openshift.withCluster() { - openshift.withProject() { - // Cleanup all test environments that were created 1 hour ago in case of failures of previous cleanups. - c3i.cleanup(script: this, 'krb5', 'umb', 'koji', 'mbs') - } - } - } - } - post { - failure { - echo "Cleanup of old environments FAILED" - } - } - } - stage('Route suffix') { - when { - expression { !env.PAAS_DOMAIN } - } - steps { - script { - openshift.withCluster() { - openshift.withProject(env.PIPELINE_ID) { - def testroute = openshift.create('route', 'edge', 'test', '--service=test', '--port=8080') - def testhost = testroute.object().spec.host - env.PAAS_DOMAIN = testhost.minus("test-${env.PIPELINE_ID}.") - testroute.delete() - } - } - } - } - post { - success { - echo "Routes end with ${env.PAAS_DOMAIN}" - } - } - } - stage('Deploy test environment') { - steps { - script { - openshift.withCluster() { - openshift.withProject(params.PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE) { - c3i.buildAndWait(script: this, objs: "bc/pipeline-as-a-service", - '-e', "DEFAULT_IMAGE_TAG=${env.ENVIRONMENT}", - '-e', "PIPELINE_ID=${env.PIPELINE_ID}", - '-e', "WAIVERDB_IMAGE=", - '-e', "C3IAAS_PROJECT=", - '-e', "RESULTSDB_IMAGE=", - '-e', "RESULTSDB_UPDATER_IMAGE=", - '-e', "GREENWAVE_IMAGE=", - '-e', "DATAGREPPER_IMAGE=", - '-e', "DATANOMMER_IMAGE=", - '-e', "MBS_BACKEND_IMAGE=${env.MBS_BACKEND_IMAGE}", - '-e', "MBS_FRONTEND_IMAGE=${env.MBS_FRONTEND_IMAGE}", - '-e', "PAAS_DOMAIN=${env.PAAS_DOMAIN}" - ) - } - } - } - } - } - stage('Run tests') { - steps { - script { - def testcases - if (params.TESTCASES) { - if (params.TESTCASES == 'skip') { - testcases = [] - echo 'Skipping integration tests' - } else { - testcases = params.TESTCASES.split() - echo "Using specified list of testcases: ${testcases}" - } - } else { - testcases = findFiles(glob: 'openshift/integration/koji/pipelines/tests/*.groovy').collect { - it.name.minus('.groovy') - } - echo "Using all available testcases: ${testcases}" - } - testcases.each { testcase -> - env.CURRENT_TESTCASE = testcase - echo "Running testcase ${testcase}..." - def test = load "openshift/integration/koji/pipelines/tests/${testcase}.groovy" - test.runTests() - } - } - } - post { - success { - echo "All tests successful" - } - failure { - echo "Testcase ${env.CURRENT_TESTCASE} FAILED" - } - } - } - } - post { - success { - script { - params.TEST_IMAGES.split(',').each { - sendToResultsDB(it, 'passed') - } - } - } - failure { - script { - params.TEST_IMAGES.split(',').each { - sendToResultsDB(it, 'failed') - } - openshift.withCluster() { - openshift.withProject(params.TEST_NAMESPACE) { - echo 'Getting logs from all deployments...' - openshift.selector('pods', ['c3i.redhat.com/pipeline': env.PIPELINE_ID]).logs('--tail 100') - } - } - } - } - cleanup { - script { - if (params.CLEANUP == 'true' && !params.TEST_NAMESPACE) { - openshift.withCluster() { - /* Tear down everything we just created */ - echo 'Tearing down test resources...' - openshift.selector('all,pvc,configmap,secret', - ['c3i.redhat.com/pipeline': env.PIPELINE_ID]).delete('--ignore-not-found=true') - } - } else { - echo 'Skipping cleanup' - } - } - } - } -} -def sendToResultsDB(imageRef, status) { - if (!params.MESSAGING_PROVIDER) { - echo "Message bus is not set. Skipping send of:\nimageRef: ${imageRef}\nstatus: ${status}" - return - } - def (repourl, digest) = imageRef.tokenize('@') - def (registry, reponame) = repourl.split('/', 2) - def image = reponame.split('/').last() - def sendResult = sendCIMessage \ - providerName: params.MESSAGING_PROVIDER, \ - overrides: [topic: 'VirtualTopic.eng.ci.container-image.test.complete'], \ - messageType: 'Custom', \ - messageProperties: '', \ - messageContent: """ - { - "ci": { - "name": "C3I Jenkins", - "team": "DevOps", - "url": "${env.JENKINS_URL}", - "docs": "https://pagure.io/fm-orchestrator/blob/master/f/openshift/integration/koji", - "irc": "#pnt-devops-dev", - "email": "pnt-factory2-devel@redhat.com", - "environment": "${params.ENVIRONMENT}" - }, - "run": { - "url": "${env.BUILD_URL}", - "log": "${env.BUILD_URL}/console", - "debug": "", - "rebuild": "${env.BUILD_URL}/rebuild/parametrized" - }, - "artifact": { - "type": "container-image", - "repository": "${reponame}", - "digest": "${digest}", - "nvr": "${imageRef}", - "issuer": "c3i-jenkins", - "scratch": ${params.IMAGE_IS_SCRATCH}, - "id": "${image}@${digest}" - }, - "system": - [{ - "os": "${params.JENKINS_AGENT_IMAGE}", - "provider": "openshift", - "architecture": "x86_64" - }], - "type": "integration", - "category": "${params.ENVIRONMENT}", - "status": "${status}", - "xunit": "", - "generated_at": "${new Date().format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone('UTC'))}", - "namespace": "c3i", - "version": "0.1.0" - } - """ - if (sendResult.getMessageId()) { - // echo sent message id and content - echo 'Successfully sent the test result to ResultsDB.' - echo "Message ID: ${sendResult.getMessageId()}" - echo "Message content: ${sendResult.getMessageContent()}" - } else { - echo 'Failed to sent the test result to ResultsDB.' - } -} diff --git a/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.Jenkinsfile b/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.Jenkinsfile new file mode 100644 index 00000000..8b9766d4 --- /dev/null +++ b/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.Jenkinsfile @@ -0,0 +1,103 @@ +{% include "snippets/c3i-library.groovy" %} +pipeline { + {% include "snippets/default-agent.groovy" %} + options { + timestamps() + timeout(time: 60, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '10')) + } + environment { + PIPELINE_NAMESPACE = readFile('/run/secrets/kubernetes.io/serviceaccount/namespace').trim() + PAGURE_URL = "${PAGURE_URL}" + PAGURE_API = "${env.PAGURE_URL}/api/0" + PAGURE_REPO_NAME = "${PAGURE_REPO_NAME}" + PAGURE_REPO_IS_FORK = "${PAGURE_REPO_IS_FORK}" + PAGURE_POLLING_FOR_PR = "${PAGURE_POLLING_FOR_PR}" + PAGURE_REPO_HOME = "${env.PAGURE_URL}${env.PAGURE_REPO_IS_FORK == 'true' ? '/fork' : ''}/${env.PAGURE_REPO_NAME}" + GIT_URL = "${env.PAGURE_URL}/${env.PAGURE_REPO_IS_FORK == 'true' ? 'forks/' : ''}${env.PAGURE_REPO_NAME}.git" + PREMERGE_JOB_NAME = "${PREMERGE_JOB_NAME}" + POSTMERGE_JOB_NAME = "${POSTMERGE_JOB_NAME}" + } + triggers { pollSCM("${PAGURE_POLLING_SCHEDULE}") } + stages { + stage('Prepare') { + agent { label 'master' } + steps { + script { + def polled = env.PAGURE_POLLING_FOR_PR == 'true' ? 'pull/*/head' : "${PAGURE_POLLED_BRANCH}" + // Need to prefix the rev with origin/ for pollSCM to work correctly + def rev = "origin/${polled}" + def scmVars = c3i.clone(repo: env.GIT_URL, branch: polled, rev: rev) + env.GIT_COMMIT = scmVars.GIT_COMMIT + // setting build display name + def prefix = 'origin/' + def branch = scmVars.GIT_BRANCH.startsWith(prefix) ? scmVars.GIT_BRANCH.substring(prefix.size()) + : scmVars.GIT_BRANCH // origin/pull/1234/head -> pull/1234/head, origin/master -> master + env.MBS_GIT_BRANCH = branch + echo "Build on branch=${env.MBS_GIT_BRANCH}, commit=${env.GIT_COMMIT}" + if (env.PAGURE_POLLING_FOR_PR == 'false') { + currentBuild.displayName = "${env.MBS_GIT_BRANCH}: ${env.GIT_COMMIT.substring(0, 7)}" + currentBuild.description = """${currentBuild.displayName}""" + } else if (env.PAGURE_POLLING_FOR_PR == 'true' && branch ==~ /^pull\/[0-9]+\/head$/) { + env.PR_NO = branch.split('/')[1] + def prInfo = pagure.getPR(env.PR_NO) + if (prInfo.status == 'Open') { + env.PR_URL = "${env.PAGURE_REPO_HOME}/pull-request/${env.PR_NO}" + // To HTML syntax in build description, go to `Jenkins/Global Security/Markup Formatter` and select 'Safe HTML'. + def pagureLink = """PR#${env.PR_NO}""" + echo "Building PR #${env.PR_NO}: ${env.PR_URL}" + currentBuild.displayName = "PR#${env.PR_NO}" + currentBuild.description = pagureLink + } else { + echo "Skipping PR#${env.PR_NO} because it is ${prInfo.status}" + env.SKIP = 'true' + } + } else { // This shouldn't happen. + error("Build is aborted due to unexpected polling trigger actions.") + } + } + } + } + stage('Update pipeline jobs') { + when { + expression { + return "${PIPELINE_UPDATE_JOBS_DIR}" && env.PAGURE_POLLING_FOR_PR == 'false' && env.MBS_GIT_BRANCH == "${PAGURE_POLLED_BRANCH}" + } + } + steps { + script { + c3i.clone(repo: env.GIT_URL, branch: env.MBS_GIT_BRANCH) + dir('openshift/integration/koji/pipelines') { + sh ''' + make install JOBS_DIR="${PIPELINE_UPDATE_JOBS_DIR}" + ''' + } + } + } + } + stage('Build') { + when { + not { + environment name: 'SKIP', value: 'true' + } + } + steps { + script { + openshift.withCluster() { + echo 'Starting a MBS build run...' + def devBuild = c3i.build(script: this, + objs: "bc/${env.PAGURE_POLLING_FOR_PR == 'true' ? env.PREMERGE_JOB_NAME : env.POSTMERGE_JOB_NAME}", + '-e', "MBS_GIT_REF=${env.MBS_GIT_BRANCH}", '-e', "PAGURE_REPO_IS_FORK=${env.PAGURE_REPO_IS_FORK}", + '-e', "PAGURE_REPO_NAME=${env.PAGURE_REPO_NAME}" + ) + c3i.waitForBuildStart(script: this, build: devBuild) + def devBuildInfo = devBuild.object() + def downstreamBuildName = devBuildInfo.metadata.name + def downstreamBuildUrl = devBuildInfo.metadata.annotations['openshift.io/jenkins-build-uri'] + echo "Downstream build ${downstreamBuildName}(${downstreamBuildUrl}) started." + } + } + } + } + } +} diff --git a/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml b/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml index 2de9b369..d9aca2c5 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml +++ b/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml @@ -58,6 +58,7 @@ parameters: displayName: Name of OpenShift cloud in Jenkins master configuration required: true value: openshift +{% include "snippets/c3i-library-parameters.yaml" %} objects: - kind: ServiceAccount apiVersion: v1 @@ -88,139 +89,12 @@ objects: strategy: type: JenkinsPipeline jenkinsPipelineStrategy: + env: + - name: JENKINS_AGENT_CLOUD_NAME + value: "${JENKINS_AGENT_CLOUD_NAME}" + - name: JENKINS_AGENT_IMAGE + value: "${JENKINS_AGENT_IMAGE}" + - name: JENKINS_AGENT_SERVICE_ACCOUNT + value: "${NAME}-jenkins-slave" jenkinsfile: |- - // Don't use external Jenkinsfile here, or Jenkins will also poll on that repo and branch - library identifier: 'c3i@master', changelog: false, - retriever: modernSCM([$class: 'GitSCMSource', remote: 'https://pagure.io/c3i-library.git']) - pipeline { - agent { - kubernetes { - cloud "${JENKINS_AGENT_CLOUD_NAME}" - label "jenkins-slave-${UUID.randomUUID().toString()}" - serviceAccount "${NAME}-jenkins-slave" - defaultContainer 'jnlp' - yaml """ - apiVersion: v1 - kind: Pod - metadata: - labels: - app: "jenkins-${env.JOB_BASE_NAME.take(50)}" - factory2-pipeline-kind: "mbs-polling-to-pagure-pipeline" - factory2-pipeline-build-number: "${env.BUILD_NUMBER}" - spec: - containers: - - name: jnlp - image: "${JENKINS_AGENT_IMAGE}" - imagePullPolicy: Always - tty: true - resources: - requests: - memory: 512Mi - cpu: 300m - limits: - memory: 768Mi - cpu: 500m - """ - } - } - options { - timestamps() - timeout(time: 60, unit: 'MINUTES') - buildDiscarder(logRotator(numToKeepStr: '10')) - skipDefaultCheckout() - } - environment { - PIPELINE_NAMESPACE = readFile('/run/secrets/kubernetes.io/serviceaccount/namespace').trim() - PAGURE_URL = "${PAGURE_URL}" - PAGURE_API = "${env.PAGURE_URL}/api/0" - PAGURE_REPO_NAME = "${PAGURE_REPO_NAME}" - PAGURE_REPO_IS_FORK = "${PAGURE_REPO_IS_FORK}" - PAGURE_POLLING_FOR_PR = "${PAGURE_POLLING_FOR_PR}" - PAGURE_REPO_HOME = "${env.PAGURE_URL}${env.PAGURE_REPO_IS_FORK == 'true' ? '/fork' : ''}/${env.PAGURE_REPO_NAME}" - GIT_URL = "${env.PAGURE_URL}/${env.PAGURE_REPO_IS_FORK == 'true' ? 'forks/' : ''}${env.PAGURE_REPO_NAME}.git" - PREMERGE_JOB_NAME = "${PREMERGE_JOB_NAME}" - POSTMERGE_JOB_NAME = "${POSTMERGE_JOB_NAME}" - } - triggers { pollSCM("${PAGURE_POLLING_SCHEDULE}") } - stages { - stage('Prepare') { - agent { label 'master' } - steps { - script { - def polled = env.PAGURE_POLLING_FOR_PR == 'true' ? 'pull/*/head' : "${PAGURE_POLLED_BRANCH}" - // Need to prefix the rev with origin/ for pollSCM to work correctly - def rev = "origin/${polled}" - def scmVars = c3i.clone(repo: env.GIT_URL, branch: polled, rev: rev) - env.GIT_COMMIT = scmVars.GIT_COMMIT - // setting build display name - def prefix = 'origin/' - def branch = scmVars.GIT_BRANCH.startsWith(prefix) ? scmVars.GIT_BRANCH.substring(prefix.size()) - : scmVars.GIT_BRANCH // origin/pull/1234/head -> pull/1234/head, origin/master -> master - env.MBS_GIT_BRANCH = branch - echo "Build on branch=${env.MBS_GIT_BRANCH}, commit=${env.GIT_COMMIT}" - if (env.PAGURE_POLLING_FOR_PR == 'false') { - currentBuild.displayName = "${env.MBS_GIT_BRANCH}: ${env.GIT_COMMIT.substring(0, 7)}" - currentBuild.description = """${currentBuild.displayName}""" - } else if (env.PAGURE_POLLING_FOR_PR == 'true' && branch ==~ /^pull\/[0-9]+\/head$/) { - env.PR_NO = branch.split('/')[1] - def prInfo = pagure.getPR(env.PR_NO) - if (prInfo.status == 'Open') { - env.PR_URL = "${env.PAGURE_REPO_HOME}/pull-request/${env.PR_NO}" - // To HTML syntax in build description, go to `Jenkins/Global Security/Markup Formatter` and select 'Safe HTML'. - def pagureLink = """PR#${env.PR_NO}""" - echo "Building PR #${env.PR_NO}: ${env.PR_URL}" - currentBuild.displayName = "PR#${env.PR_NO}" - currentBuild.description = pagureLink - } else { - echo "Skipping PR#${env.PR_NO} because it is ${prInfo.status}" - env.SKIP = 'true' - } - } else { // This shouldn't happen. - error("Build is aborted due to unexpected polling trigger actions.") - } - } - } - } - stage('Update pipeline jobs') { - when { - expression { - return "${PIPELINE_UPDATE_JOBS_DIR}" && env.PAGURE_POLLING_FOR_PR == 'false' && env.MBS_GIT_BRANCH == "${PAGURE_POLLED_BRANCH}" - } - } - steps { - script { - c3i.clone(repo: env.GIT_URL, branch: env.MBS_GIT_BRANCH) - dir('openshift/integration/koji/pipelines') { - sh ''' - make install JOBS_DIR="${PIPELINE_UPDATE_JOBS_DIR}" - ''' - } - } - } - } - stage('Build') { - when { - not { - environment name: 'SKIP', value: 'true' - } - } - steps { - script { - openshift.withCluster() { - echo 'Starting a MBS build run...' - def devBuild = c3i.build(script: this, - objs: "bc/${env.PAGURE_POLLING_FOR_PR == 'true' ? env.PREMERGE_JOB_NAME : env.POSTMERGE_JOB_NAME}", - '-e', "MBS_GIT_REF=${env.MBS_GIT_BRANCH}", '-e', "PAGURE_REPO_IS_FORK=${env.PAGURE_REPO_IS_FORK}", - '-e', "PAGURE_REPO_NAME=${env.PAGURE_REPO_NAME}" - ) - c3i.waitForBuildStart(script: this, build: devBuild) - def devBuildInfo = devBuild.object() - def downstreamBuildName = devBuildInfo.metadata.name - def downstreamBuildUrl = devBuildInfo.metadata.annotations['openshift.io/jenkins-build-uri'] - echo "Downstream build ${downstreamBuildName}(${downstreamBuildUrl}) started." - } - } - } - } - } - } + {% filter indent(width=10) %}{% include "mbs-polling-pagure.Jenkinsfile" %}{% endfilter %} diff --git a/openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.Jenkinsfile b/openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.Jenkinsfile index eaa4b718..001760de 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.Jenkinsfile +++ b/openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.Jenkinsfile @@ -1,110 +1,56 @@ -// Use scripted syntax because CIBuildTrigger currently doesn't support the declarative syntax -properties([ - buildDiscarder(logRotator(numToKeepStr: '10')), - pipelineTriggers([ - // example: https://github.com/jenkinsci/jms-messaging-plugin/blob/9b9387c3a52f037ba0d019c2ebcf2a2796fc6397/src/test/java/com/redhat/jenkins/plugins/ci/integration/AmqMessagingPluginIntegrationTest.java - [$class: 'CIBuildTrigger', - providerData: [$class: 'ActiveMQSubscriberProviderData', - name: params.MESSAGING_PROVIDER, - overrides: [topic: params.MESSAGING_TOPIC], - selector: "repo = '${params.TRACKED_CONTAINER_REPO}' AND action IN ('added', 'updated') AND tag = '${params.TRACKED_TAG}'", - ], - ], - ]), -]) - -if (!params.CI_MESSAGE) { - echo 'This build is not started by a CI message. Only configurations were done.' - return -} -library identifier: 'c3i@master', changelog: false, - retriever: modernSCM([$class: 'GitSCMSource', remote: 'https://pagure.io/c3i-library.git']) -def label = "jenkins-slave-${UUID.randomUUID().toString()}" -podTemplate( - cloud: "${params.JENKINS_AGENT_CLOUD_NAME}", - label: label, - serviceAccount: "${env.JENKINS_AGENT_SERVICE_ACCOUNT}", - yaml: """ - apiVersion: v1 - kind: Pod - metadata: - labels: - app: "jenkins-${env.JOB_BASE_NAME.take(50)}" - factory2-pipeline-kind: "mbs-repotracker-trigger" - factory2-pipeline-build-number: "${env.BUILD_NUMBER}" - spec: - containers: - - name: jnlp - image: ${params.JENKINS_AGENT_IMAGE} - imagePullPolicy: Always - tty: true - env: - - name: REGISTRY_CREDENTIALS - valueFrom: - secretKeyRef: - name: "${params.CONTAINER_REGISTRY_CREDENTIALS}" - key: ".dockerconfigjson" - resources: - requests: - memory: 256Mi - cpu: 200m - limits: - memory: 512Mi - cpu: 300m - """ -) { - node(label) { - stage('Allocate C3IaaS project') { - if (!(params.USE_C3IAAS == 'true' && - params.C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE && - params.C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAME)) { - echo "Not configured to use C3IaaS" - return - } - env.PIPELINE_NAMESPACE = readFile("/run/secrets/kubernetes.io/serviceaccount/namespace").trim() - env.C3IAAS_NAMESPACE = "c3i-mbs-tr-${params.TRACKED_TAG}-${env.BUILD_NUMBER}" - echo "Requesting new OpenShift project ${env.C3IAAS_NAMESPACE}..." - openshift.withCluster() { - openshift.withProject(params.C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE) { - c3i.buildAndWait(script: this, objs: "bc/${params.C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAME}", - '-e', "PROJECT_NAME=${env.C3IAAS_NAMESPACE}", - '-e', "ADMIN_GROUPS=system:serviceaccounts:${env.PIPELINE_NAMESPACE}", - '-e', "LIFETIME_IN_MINUTES=${params.C3IAAS_LIFETIME}" - ) - } - } - echo "Allocated project ${env.C3IAAS_NAMESPACE}" - } - stage('Trigger tests') { - def message = readJSON text: params.CI_MESSAGE - echo "Tag :${message.tag} is ${message.action} in ${message.repo}. New digest: ${message.digest}" - def frontendImage = "${message.repo}@${message.digest}" - // We have the digest of the current frontend image with this tag. - // Lookup the digest of the current backend image with the same tag. - if (env.REGISTRY_CREDENTIALS) { - dir ("${env.HOME}/.docker") { - writeFile file: 'config.json', text: env.REGISTRY_CREDENTIALS - } - } - def output = sh(script: "skopeo inspect docker://${params.MBS_BACKEND_REPO}:${message.tag}", returnStdout: true).trim() - def backendData = readJSON text: output - def backendImage = "${params.MBS_BACKEND_REPO}@${backendData.Digest}" - echo "Current mbs-backend image is: ${backendImage}" - echo "Triggering a job to test if ${frontendImage} and ${backendImage} meet all criteria of desired tag" - openshift.withCluster() { - openshift.withProject(params.TEST_JOB_NAMESPACE) { - def build = c3i.build(script: this, objs: "bc/${params.TEST_JOB_NAME}", - '-e', "MBS_BACKEND_IMAGE=${backendImage}", - '-e', "MBS_FRONTEND_IMAGE=${frontendImage}", - '-e', "TEST_IMAGES=${backendImage},${frontendImage}", - '-e', "IMAGE_IS_SCRATCH=false", - '-e', "TEST_NAMESPACE=${env.C3IAAS_NAMESPACE ?: ''}", - ) - c3i.waitForBuildStart(script: this, build: build) - buildInfo = build.object() - echo "Build ${buildInfo.metadata.annotations['openshift.io/jenkins-build-uri'] ?: buildInfo.metadata.name} started." +{% include "snippets/c3i-library.groovy" %} +pipeline { + {% include "snippets/default-agent.groovy" %} + options { + timestamps() + timeout(time: 120, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '10')) + } + triggers { + ciBuildTrigger( + noSquash: false, + providerList: [ + activeMQSubscriber( + name: params.MESSAGING_PROVIDER, + overrides: [topic: params.MESSAGING_TOPIC], + selector: "repo = '${params.TRACKED_CONTAINER_REPO}' AND action IN ('added', 'updated') AND tag = '${params.TRACKED_TAG}'", + ) + ] + ) + } + stages { + stage("Message Check and setup") { + steps { + script { + if (!params.CI_MESSAGE) { + error("This build is not started by a CI message. Only configurations were done.") + } + c3i.clone(repo: params.MBS_GIT_REPO, branch: params.MBS_GIT_REF) + def message = readJSON text: params.CI_MESSAGE + echo "Tag :${message.tag} is ${message.action} in ${message.repo}. New digest: ${message.digest}" + env.FRONTEND_IMAGE_REF = "${message.repo}@${message.digest}" + // We have the digest of the current frontend image with this tag. + // Lookup the digest of the current backend image with the same tag. + if (params.CONTAINER_REGISTRY_CREDENTIALS) { + dir ("${env.HOME}/.docker") { + openshift.withCluster() { + def dockerconf = openshift.selector('secret', params.CONTAINER_REGISTRY_CREDENTIALS).object().data['.dockerconfigjson'] + writeFile file: 'config.json', text: dockerconf, encoding: "Base64" + } + } + } + def output = sh(script: "skopeo inspect docker://${params.MBS_BACKEND_REPO}:${message.tag}", returnStdout: true).trim() + def backendData = readJSON text: output + env.BACKEND_IMAGE_REF = "${params.MBS_BACKEND_REPO}@${backendData.Digest}" + echo "Current mbs-backend image is: ${env.BACKEND_IMAGE_REF}" + echo "Triggering a job to test if ${env.FRONTEND_IMAGE_REF} and ${env.BACKEND_IMAGE_REF} meet all criteria of desired tag" + env.C3IAAS_PROJECT = params.C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE + env.IMAGE_IS_SCRATCH = false + env.PIPELINE_ID = "c3i-mbs-tag-${message.tag}-${message.digest[-9..-1]}" } } } + {% include "snippets/mbs-integration-test.groovy" %} } } +{% include "snippets/functions.groovy" %} diff --git a/openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.yaml b/openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.yaml index f9018ca8..6a373813 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.yaml +++ b/openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.yaml @@ -15,10 +15,12 @@ parameters: - name: MBS_GIT_REPO displayName: MBS Git repo URL description: Default MBS Git repo URL in which to run dev tests against - value: "https://pagure.io/fm-orchestrator.git" + required: true + value: https://pagure.io/fm-orchestrator.git - name: MBS_GIT_REF displayName: MBS Git repo ref description: Default MBS Git repo ref in which to run dev tests against + required: true value: master - name: TRACKED_CONTAINER_REPO displayName: Container repo to be tracked @@ -45,12 +47,10 @@ parameters: - name: MESSAGING_TOPIC displayName: Name of the topic that the trigger subscribes to value: "Consumer.rh-jenkins-ci-plugin.c3i-mbs-repotracker-trigger.VirtualTopic.eng.repotracker.container.tag.>" -- name: TEST_JOB_NAME - displayName: Name of integration test job to trigger +- name: ENVIRONMENT + displayName: environment name (dev/stage/prod) required: true -- name: TEST_JOB_NAMESPACE - displayName: Namespace in which to trigger the integration test job - required: false + value: dev - name: USE_C3IAAS displayName: >- Use C3I-as-a-Service to dynamically allocate a temporary OpenShift project for building @@ -69,6 +69,11 @@ parameters: displayName: The lifetime of the OpenShift project allocated by C3I-as-a-Service. required: true value: "120" +- name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE + displayName: The namespace where the Pipeline-as-a-Service project request BuildConfig has been defined + required: false + value: c3i +{% include "snippets/c3i-library-parameters.yaml" %} objects: - kind: ServiceAccount apiVersion: v1 @@ -96,10 +101,6 @@ objects: spec: runPolicy: "Parallel" completionDeadlineSeconds: 1800 - source: - git: - uri: "${MBS_GIT_REPO}" - ref: "${MBS_GIT_REF}" strategy: type: JenkinsPipeline jenkinsPipelineStrategy: @@ -110,6 +111,10 @@ objects: value: "${JENKINS_AGENT_IMAGE}" - name: JENKINS_AGENT_SERVICE_ACCOUNT value: "${NAME}-jenkins-slave" + - name: MBS_GIT_REPO + value: "${MBS_GIT_REPO}" + - name: MBS_GIT_REF + value: "${MBS_GIT_REF}" - name: TRACKED_CONTAINER_REPO value: "${TRACKED_CONTAINER_REPO}" - name: TRACKED_TAG @@ -118,18 +123,18 @@ objects: value: "${MBS_BACKEND_REPO}" - name: CONTAINER_REGISTRY_CREDENTIALS value: "${CONTAINER_REGISTRY_CREDENTIALS}" - - name: TEST_JOB_NAME - value: "${TEST_JOB_NAME}" - - name: TEST_JOB_NAMESPACE - value: "${TEST_JOB_NAMESPACE}" - name: USE_C3IAAS value: "${USE_C3IAAS}" + - name: ENVIRONMENT + value: "${ENVIRONMENT}" - name: C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE value: "${C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE}" - name: C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAME value: "${C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAME}" - name: C3IAAS_LIFETIME value: "${C3IAAS_LIFETIME}" + - name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE + value: "${PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE}" - name: MESSAGING_PROVIDER value: "${MESSAGING_PROVIDER}" - name: MESSAGING_TOPIC @@ -139,4 +144,5 @@ objects: value: - name: MESSAGE_HEADERS value: - jenkinsfilePath: openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.Jenkinsfile + jenkinsfile: | + {% filter indent(width=10) %}{% include "mbs-repotracker-trigger.Jenkinsfile" %}{% endfilter %} diff --git a/openshift/integration/koji/pipelines/templates/snippets/c3i-library-parameters.yaml b/openshift/integration/koji/pipelines/templates/snippets/c3i-library-parameters.yaml new file mode 100644 index 00000000..5f96d609 --- /dev/null +++ b/openshift/integration/koji/pipelines/templates/snippets/c3i-library-parameters.yaml @@ -0,0 +1,8 @@ +- name: C3I_LIB_URL + displayName: C3I library git url + required: true + value: "https://pagure.io/c3i-library.git" +- name: C3I_LIB_BRANCH + displayName: C3I library branch + required: true + value: "master" diff --git a/openshift/integration/koji/pipelines/templates/snippets/c3i-library.groovy b/openshift/integration/koji/pipelines/templates/snippets/c3i-library.groovy new file mode 100644 index 00000000..e752c819 --- /dev/null +++ b/openshift/integration/koji/pipelines/templates/snippets/c3i-library.groovy @@ -0,0 +1,2 @@ +library identifier: "c3i@${C3I_LIB_BRANCH}", changelog: false, + retriever: modernSCM([$class: 'GitSCMSource', remote: "${C3I_LIB_URL}"]) diff --git a/openshift/integration/koji/pipelines/templates/snippets/default-agent.groovy b/openshift/integration/koji/pipelines/templates/snippets/default-agent.groovy new file mode 100644 index 00000000..919259dc --- /dev/null +++ b/openshift/integration/koji/pipelines/templates/snippets/default-agent.groovy @@ -0,0 +1,29 @@ +agent { + kubernetes { + cloud "${params.JENKINS_AGENT_CLOUD_NAME}" + label "jenkins-slave-${UUID.randomUUID().toString()}" + serviceAccount "${params.JENKINS_AGENT_SERVICE_ACCOUNT}" + defaultContainer 'jnlp' + yaml """ + apiVersion: v1 + kind: Pod + metadata: + labels: + app: "jenkins-${env.JOB_BASE_NAME.take(50).endsWith('-') ? env.JOB_BASE_NAME.take(49): env.JOB_BASE_NAME.take(50)}" + factory2-pipeline-build-number: "${env.BUILD_NUMBER}" + spec: + containers: + - name: jnlp + image: "${params.JENKINS_AGENT_IMAGE}" + imagePullPolicy: Always + tty: true + resources: + requests: + memory: 512Mi + cpu: 300m + limits: + memory: 768Mi + cpu: 500m + """ + } +} diff --git a/openshift/integration/koji/pipelines/templates/snippets/functions.groovy b/openshift/integration/koji/pipelines/templates/snippets/functions.groovy new file mode 100644 index 00000000..1f12fc92 --- /dev/null +++ b/openshift/integration/koji/pipelines/templates/snippets/functions.groovy @@ -0,0 +1,64 @@ +def sendToResultsDB(imageRef, status) { + if (!params.MESSAGING_PROVIDER) { + echo "Message bus is not set. Skipping send of:\nimageRef: ${imageRef}\nstatus: ${status}" + return + } + def (repourl, digest) = imageRef.tokenize('@') + def (registry, reponame) = repourl.split('/', 2) + def image = reponame.split('/').last() + def sendResult = sendCIMessage \ + providerName: params.MESSAGING_PROVIDER, \ + overrides: [topic: 'VirtualTopic.eng.ci.container-image.test.complete'], \ + messageType: 'Custom', \ + messageProperties: '', \ + messageContent: """ + { + "ci": { + "name": "C3I Jenkins", + "team": "DevOps", + "url": "${env.JENKINS_URL}", + "docs": "https://pagure.io/fm-orchestrator/blob/master/f/openshift/integration/koji", + "irc": "#pnt-devops-dev", + "email": "pnt-factory2-devel@redhat.com", + "environment": "${params.ENVIRONMENT}" + }, + "run": { + "url": "${env.BUILD_URL}", + "log": "${env.BUILD_URL}/console", + "debug": "", + "rebuild": "${env.BUILD_URL}/rebuild/parametrized" + }, + "artifact": { + "type": "container-image", + "repository": "${reponame}", + "digest": "${digest}", + "nvr": "${imageRef}", + "issuer": "c3i-jenkins", + "scratch": ${env.IMAGE_IS_SCRATCH}, + "id": "${image}@${digest}" + }, + "system": + [{ + "os": "${params.JENKINS_AGENT_IMAGE}", + "provider": "openshift", + "architecture": "x86_64" + }], + "type": "integration", + "category": "${params.ENVIRONMENT}", + "status": "${status}", + "xunit": "", + "generated_at": "${new Date().format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone('UTC'))}", + "namespace": "c3i", + "version": "0.1.0" + } + """ + if (sendResult.getMessageId()) { + // echo sent message id and content + echo 'Successfully sent the test result to ResultsDB.' + echo "Message ID: ${sendResult.getMessageId()}" + echo "Message content: ${sendResult.getMessageContent()}" + } else { + echo 'Failed to sent the test result to ResultsDB.' + } +} + diff --git a/openshift/integration/koji/pipelines/templates/snippets/get_paas_domain.groovy b/openshift/integration/koji/pipelines/templates/snippets/get_paas_domain.groovy new file mode 100644 index 00000000..9733732c --- /dev/null +++ b/openshift/integration/koji/pipelines/templates/snippets/get_paas_domain.groovy @@ -0,0 +1,14 @@ +if (!env.TRIGGER_NAMESPACE) { + env.TRIGGER_NAMESPACE = readFile("/run/secrets/kubernetes.io/serviceaccount/namespace").trim() +} +if(!env.PAAS_DOMAIN) { + openshift.withCluster() { + openshift.withProject(env.TRIGGER_NAMESPACE) { + def testroute = openshift.create('route', 'edge', "test-${env.BUILD_NUMBER}", '--service=test', '--port=8080') + def testhost = testroute.object().spec.host + env.PAAS_DOMAIN = testhost.minus("test-${env.BUILD_NUMBER}-${env.TRIGGER_NAMESPACE}.") + testroute.delete() + } + } + echo "Routes end with ${env.PAAS_DOMAIN}" +} diff --git a/openshift/integration/koji/pipelines/templates/snippets/mbs-integration-test.groovy b/openshift/integration/koji/pipelines/templates/snippets/mbs-integration-test.groovy new file mode 100644 index 00000000..d71bca59 --- /dev/null +++ b/openshift/integration/koji/pipelines/templates/snippets/mbs-integration-test.groovy @@ -0,0 +1,103 @@ +stage('Run integration tests') { + stages { + stage('Deploy test environment') { + steps { + script { + {% include "snippets/get_paas_domain.groovy" %} + if (!env.PIPELINE_ID) { + env.PIPELINE_ID = "c3i-mbs-${UUID.randomUUID().toString().take(8)}" + } + openshift.withCluster() { + openshift.withProject(params.PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE) { + c3i.buildAndWait(script: this, objs: "bc/pipeline-as-a-service", + '-e', "DEFAULT_IMAGE_TAG=${env.ENVIRONMENT}", + '-e', "PIPELINE_ID=${env.PIPELINE_ID}", + '-e', "WAIVERDB_IMAGE=", + '-e', "C3IAAS_PROJECT=${env.C3IAAS_PROJECT ?: ''}", + '-e', "RESULTSDB_IMAGE=", + '-e', "RESULTSDB_UPDATER_IMAGE=", + '-e', "GREENWAVE_IMAGE=", + '-e', "DATAGREPPER_IMAGE=", + '-e', "DATANOMMER_IMAGE=", + '-e', "MBS_BACKEND_IMAGE=${env.BACKEND_IMAGE_REF}", + '-e', "MBS_FRONTEND_IMAGE=${env.FRONTEND_IMAGE_REF}", + '-e', "PAAS_DOMAIN=${env.PAAS_DOMAIN}" + ) + } + } + } + } + } + stage('Run tests') { + steps { + script { + def testcases + if (params.TESTCASES) { + if (params.TESTCASES == 'skip') { + testcases = [] + echo 'Skipping integration tests' + } else { + testcases = params.TESTCASES.split() + echo "Using specified list of testcases: ${testcases}" + } + } else { + testcases = findFiles(glob: 'openshift/integration/koji/pipelines/tests/*.groovy').collect { + it.name.minus('.groovy') + } + echo "Using all available testcases: ${testcases}" + } + testcases.each { testcase -> + env.CURRENT_TESTCASE = testcase + echo "Running testcase ${testcase}..." + def test = load "openshift/integration/koji/pipelines/tests/${testcase}.groovy" + test.runTests() + } + } + } + post { + success { + echo "All tests successful" + script { + [env.BACKEND_IMAGE_REF, env.FRONTEND_IMAGE_REF].each { + sendToResultsDB(it, 'passed') + } + } + } + failure { + echo "Testcase ${env.CURRENT_TESTCASE} FAILED" + } + } + } + } + post { + failure { + script { + [env.BACKEND_IMAGE_REF, env.FRONTEND_IMAGE_REF].each { + sendToResultsDB(it, 'failed') + } + openshift.withCluster() { + openshift.withProject(env.PIPELINE_ID) { + echo 'Getting logs from all deployments...' + openshift.selector('pods', ['c3i.redhat.com/pipeline': env.PIPELINE_ID]).logs('--tail 100') + } + } + } + } + cleanup { + script { + if (params.CLEANUP == 'true' && params.USE_C3IAAS != 'true') { + openshift.withCluster() { + openshift.withProject(env.PIPELINE_ID) { + /* Tear down everything we just created */ + echo 'Tearing down test resources...' + openshift.selector('all,pvc,configmap,secret', + ['c3i.redhat.com/pipeline': env.PIPELINE_ID]).delete('--ignore-not-found=true') + } + } + } else { + echo 'Skipping cleanup' + } + } + } + } +}