From a97f6ce9ff9cf75e5f556d176d0799436fbb031a Mon Sep 17 00:00:00 2001 From: Michal Kovarik Date: Fri, 6 Dec 2019 08:40:39 +0100 Subject: [PATCH] Use jinja templates to provide 'full-jobs' Reduce number of jobs from 20 to 12 by merging them and using templating to avoid code duplicity. Jenkinsfile are directly put into jobs as a script - not pulled from repository - avoiding issue with not clean workspace. Secrets are taken from openshift using service account intead of refering them in environment which caused that secret had to exist and could be empty. Job execution is straitforward - one jobs is repsonsible for each process, only triggered jobs are c3iaas request and pipeline-as-a-service job, everything else is managed by one job. --- .../koji/containers/jenkins-slave/Dockerfile | 2 +- openshift/integration/koji/pipelines/Makefile | 2 +- .../mbs-backend-greenwave-promote-to-prod.env | 4 +- ...mbs-backend-greenwave-promote-to-stage.env | 4 +- .../jobs/mbs-backend-promoting-to-prod.env | 5 - .../jobs/mbs-backend-promoting-to-prod.tmpl | 1 - .../jobs/mbs-backend-promoting-to-stage.env | 5 - .../jobs/mbs-backend-promoting-to-stage.tmpl | 1 - .../jobs/mbs-dev-integration-test.env | 1 - .../jobs/mbs-dev-integration-test.tmpl | 1 - ...mbs-frontend-greenwave-promote-to-prod.env | 4 +- ...bs-frontend-greenwave-promote-to-stage.env | 4 +- .../jobs/mbs-frontend-promoting-to-prod.env | 5 - .../jobs/mbs-frontend-promoting-to-prod.tmpl | 1 - .../jobs/mbs-frontend-promoting-to-stage.env | 5 - .../jobs/mbs-frontend-promoting-to-stage.tmpl | 1 - .../jobs/mbs-prod-integration-test.env | 2 - .../jobs/mbs-prod-integration-test.tmpl | 1 - .../jobs/mbs-stage-integration-test.env | 2 - .../jobs/mbs-stage-integration-test.tmpl | 1 - .../jobs/mbs-trigger-on-latest-tag.env | 2 +- .../jobs/mbs-trigger-on-stage-tag.env | 2 +- .../templates/mbs-build-template.yaml | 31 +- .../pipelines/templates/mbs-build.Jenkinsfile | 90 ++---- .../mbs-greenwave-trigger.Jenkinsfile | 193 ++++++++----- .../templates/mbs-greenwave-trigger.yaml | 35 ++- .../mbs-image-promotion-template.yaml | 120 -------- .../templates/mbs-image-promotion.Jenkinsfile | 125 -------- .../mbs-integration-test-template.yaml | 143 ---------- .../mbs-integration-test.Jenkinsfile | 267 ------------------ .../templates/mbs-polling-pagure.Jenkinsfile | 103 +++++++ .../templates/mbs-polling-pagure.yaml | 144 +--------- .../mbs-repotracker-trigger.Jenkinsfile | 156 ++++------ .../templates/mbs-repotracker-trigger.yaml | 36 ++- .../snippets/c3i-library-parameters.yaml | 8 + .../templates/snippets/c3i-library.groovy | 2 + .../templates/snippets/default-agent.groovy | 29 ++ .../templates/snippets/functions.groovy | 64 +++++ .../templates/snippets/get_paas_domain.groovy | 14 + .../snippets/mbs-integration-test.groovy | 103 +++++++ 40 files changed, 596 insertions(+), 1123 deletions(-) delete mode 100644 openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-prod.env delete mode 100644 openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-prod.tmpl delete mode 100644 openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-stage.env delete mode 100644 openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-stage.tmpl delete mode 100644 openshift/integration/koji/pipelines/jobs/mbs-dev-integration-test.env delete mode 100644 openshift/integration/koji/pipelines/jobs/mbs-dev-integration-test.tmpl delete mode 100644 openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-prod.env delete mode 100644 openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-prod.tmpl delete mode 100644 openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-stage.env delete mode 100644 openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-stage.tmpl delete mode 100644 openshift/integration/koji/pipelines/jobs/mbs-prod-integration-test.env delete mode 100644 openshift/integration/koji/pipelines/jobs/mbs-prod-integration-test.tmpl delete mode 100644 openshift/integration/koji/pipelines/jobs/mbs-stage-integration-test.env delete mode 100644 openshift/integration/koji/pipelines/jobs/mbs-stage-integration-test.tmpl delete mode 100644 openshift/integration/koji/pipelines/templates/mbs-image-promotion-template.yaml delete mode 100644 openshift/integration/koji/pipelines/templates/mbs-image-promotion.Jenkinsfile delete mode 100644 openshift/integration/koji/pipelines/templates/mbs-integration-test-template.yaml delete mode 100644 openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile create mode 100644 openshift/integration/koji/pipelines/templates/mbs-polling-pagure.Jenkinsfile create mode 100644 openshift/integration/koji/pipelines/templates/snippets/c3i-library-parameters.yaml create mode 100644 openshift/integration/koji/pipelines/templates/snippets/c3i-library.groovy create mode 100644 openshift/integration/koji/pipelines/templates/snippets/default-agent.groovy create mode 100644 openshift/integration/koji/pipelines/templates/snippets/functions.groovy create mode 100644 openshift/integration/koji/pipelines/templates/snippets/get_paas_domain.groovy create mode 100644 openshift/integration/koji/pipelines/templates/snippets/mbs-integration-test.groovy diff --git a/openshift/integration/koji/containers/jenkins-slave/Dockerfile b/openshift/integration/koji/containers/jenkins-slave/Dockerfile index ae8666f8..2c8167f8 100644 --- a/openshift/integration/koji/containers/jenkins-slave/Dockerfile +++ b/openshift/integration/koji/containers/jenkins-slave/Dockerfile @@ -32,7 +32,7 @@ USER root RUN ${DNF_CMD} install -y \ java-1.8.0-openjdk nss_wrapper gettext git \ - tar gzip skopeo wget make bind-utils \ + tar gzip skopeo wget make bind-utils python3-jinja2-cli \ origin-clients \ # Jenkins pipeline 'sh' steps seem to require ps procps-ng \ diff --git a/openshift/integration/koji/pipelines/Makefile b/openshift/integration/koji/pipelines/Makefile index de353e26..cfb8e06e 100644 --- a/openshift/integration/koji/pipelines/Makefile +++ b/openshift/integration/koji/pipelines/Makefile @@ -23,7 +23,7 @@ install: @for job in $(JOBS); do \ echo "[PIPELINE] Updating pipeline job \"$${job}\"..." ; \ template_file=$$(cat ./$(JOBS_DIR)/$${job}.tmpl); \ - $(OC_CMD) process --local -f ./$(TEMPLATES_DIR)/$${template_file} \ + jinja2 ./$(TEMPLATES_DIR)/$${template_file} | $(OC_CMD) process --local -f - \ --param-file ./$(JOBS_DIR)/$${job}.env | $(OC_CMD) apply -f -; \ echo "[PIPELINE] Pipeline job \"$${job}\" updated" ; \ done diff --git a/openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-prod.env b/openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-prod.env index e03e3abb..2bc83735 100644 --- a/openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-prod.env +++ b/openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-prod.env @@ -4,4 +4,6 @@ SUBJECT_IDENTIFIER_REGEX=^factory2/mbs-backend@sha256: SOURCE_CONTAINER_REPO=quay.io/factory2/mbs-backend TARGET_TAG=prod MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-backend-greenwave-promote-to-prod.VirtualTopic.eng.greenwave.decision.update -IMAGE_PROMOTION_JOB=mbs-backend-promoting-to-prod +PROMOTING_DESTINATIONS=quay.io/factory2/mbs-backend +TAG_INTO_IMAGESTREAM=true +DEST_IMAGESTREAM_NAME=mbs-backend diff --git a/openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-stage.env b/openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-stage.env index 2a007d11..e1de485f 100644 --- a/openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-stage.env +++ b/openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-stage.env @@ -4,4 +4,6 @@ SUBJECT_IDENTIFIER_REGEX=^factory2/mbs-backend@sha256: SOURCE_CONTAINER_REPO=quay.io/factory2/mbs-backend TARGET_TAG=stage MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-backend-greenwave-promote-to-stage.VirtualTopic.eng.greenwave.decision.update -IMAGE_PROMOTION_JOB=mbs-backend-promoting-to-stage +PROMOTING_DESTINATIONS=quay.io/factory2/mbs-backend +TAG_INTO_IMAGESTREAM=true +DEST_IMAGESTREAM_NAME=mbs-backend diff --git a/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-prod.env b/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-prod.env deleted file mode 100644 index 44ee43d6..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-prod.env +++ /dev/null @@ -1,5 +0,0 @@ -NAME=mbs-backend-promoting-to-prod -PROMOTING_DESTINATIONS=quay.io/factory2/mbs-backend -DEST_TAG=prod -TAG_INTO_IMAGESTREAM=true -DEST_IMAGESTREAM_NAME=mbs-backend diff --git a/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-prod.tmpl b/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-prod.tmpl deleted file mode 100644 index cd70fa7a..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-prod.tmpl +++ /dev/null @@ -1 +0,0 @@ -mbs-image-promotion-template.yaml diff --git a/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-stage.env b/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-stage.env deleted file mode 100644 index 9439ca67..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-stage.env +++ /dev/null @@ -1,5 +0,0 @@ -NAME=mbs-backend-promoting-to-stage -PROMOTING_DESTINATIONS=quay.io/factory2/mbs-backend -DEST_TAG=stage -TAG_INTO_IMAGESTREAM=true -DEST_IMAGESTREAM_NAME=mbs-backend diff --git a/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-stage.tmpl b/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-stage.tmpl deleted file mode 100644 index cd70fa7a..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-stage.tmpl +++ /dev/null @@ -1 +0,0 @@ -mbs-image-promotion-template.yaml diff --git a/openshift/integration/koji/pipelines/jobs/mbs-dev-integration-test.env b/openshift/integration/koji/pipelines/jobs/mbs-dev-integration-test.env deleted file mode 100644 index 199953f1..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-dev-integration-test.env +++ /dev/null @@ -1 +0,0 @@ -NAME=mbs-dev-integration-test diff --git a/openshift/integration/koji/pipelines/jobs/mbs-dev-integration-test.tmpl b/openshift/integration/koji/pipelines/jobs/mbs-dev-integration-test.tmpl deleted file mode 100644 index 467c9eea..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-dev-integration-test.tmpl +++ /dev/null @@ -1 +0,0 @@ -mbs-integration-test-template.yaml diff --git a/openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-prod.env b/openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-prod.env index 6d80b8fa..55af128e 100644 --- a/openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-prod.env +++ b/openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-prod.env @@ -4,4 +4,6 @@ SUBJECT_IDENTIFIER_REGEX=^factory2/mbs-frontend@sha256: SOURCE_CONTAINER_REPO=quay.io/factory2/mbs-frontend TARGET_TAG=prod MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-frontend-greenwave-promote-to-prod.VirtualTopic.eng.greenwave.decision.update -IMAGE_PROMOTION_JOB=mbs-frontend-promoting-to-prod +PROMOTING_DESTINATIONS=quay.io/factory2/mbs-frontend +TAG_INTO_IMAGESTREAM=true +DEST_IMAGESTREAM_NAME=mbs-frontend diff --git a/openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-stage.env b/openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-stage.env index a1be5810..80022fb8 100644 --- a/openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-stage.env +++ b/openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-stage.env @@ -4,4 +4,6 @@ SUBJECT_IDENTIFIER_REGEX=^factory2/mbs-frontend@sha256: SOURCE_CONTAINER_REPO=quay.io/factory2/mbs-frontend TARGET_TAG=stage MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-frontend-greenwave-promote-to-stage.VirtualTopic.eng.greenwave.decision.update -IMAGE_PROMOTION_JOB=mbs-frontend-promoting-to-stage +PROMOTING_DESTINATIONS=quay.io/factory2/mbs-frontend +TAG_INTO_IMAGESTREAM=true +DEST_IMAGESTREAM_NAME=mbs-frontend diff --git a/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-prod.env b/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-prod.env deleted file mode 100644 index 40271273..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-prod.env +++ /dev/null @@ -1,5 +0,0 @@ -NAME=mbs-frontend-promoting-to-prod -PROMOTING_DESTINATIONS=quay.io/factory2/mbs-frontend -DEST_TAG=prod -TAG_INTO_IMAGESTREAM=true -DEST_IMAGESTREAM_NAME=mbs-frontend diff --git a/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-prod.tmpl b/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-prod.tmpl deleted file mode 100644 index cd70fa7a..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-prod.tmpl +++ /dev/null @@ -1 +0,0 @@ -mbs-image-promotion-template.yaml diff --git a/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-stage.env b/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-stage.env deleted file mode 100644 index 2c3d0e4f..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-stage.env +++ /dev/null @@ -1,5 +0,0 @@ -NAME=mbs-frontend-promoting-to-stage -PROMOTING_DESTINATIONS=quay.io/factory2/mbs-frontend -DEST_TAG=stage -TAG_INTO_IMAGESTREAM=true -DEST_IMAGESTREAM_NAME=mbs-frontend diff --git a/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-stage.tmpl b/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-stage.tmpl deleted file mode 100644 index cd70fa7a..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-stage.tmpl +++ /dev/null @@ -1 +0,0 @@ -mbs-image-promotion-template.yaml diff --git a/openshift/integration/koji/pipelines/jobs/mbs-prod-integration-test.env b/openshift/integration/koji/pipelines/jobs/mbs-prod-integration-test.env deleted file mode 100644 index c9d34e1e..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-prod-integration-test.env +++ /dev/null @@ -1,2 +0,0 @@ -NAME=mbs-prod-integration-test -ENVIRONMENT=prod diff --git a/openshift/integration/koji/pipelines/jobs/mbs-prod-integration-test.tmpl b/openshift/integration/koji/pipelines/jobs/mbs-prod-integration-test.tmpl deleted file mode 100644 index 467c9eea..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-prod-integration-test.tmpl +++ /dev/null @@ -1 +0,0 @@ -mbs-integration-test-template.yaml diff --git a/openshift/integration/koji/pipelines/jobs/mbs-stage-integration-test.env b/openshift/integration/koji/pipelines/jobs/mbs-stage-integration-test.env deleted file mode 100644 index f94244f7..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-stage-integration-test.env +++ /dev/null @@ -1,2 +0,0 @@ -NAME=mbs-stage-integration-test -ENVIRONMENT=stage diff --git a/openshift/integration/koji/pipelines/jobs/mbs-stage-integration-test.tmpl b/openshift/integration/koji/pipelines/jobs/mbs-stage-integration-test.tmpl deleted file mode 100644 index 467c9eea..00000000 --- a/openshift/integration/koji/pipelines/jobs/mbs-stage-integration-test.tmpl +++ /dev/null @@ -1 +0,0 @@ -mbs-integration-test-template.yaml diff --git a/openshift/integration/koji/pipelines/jobs/mbs-trigger-on-latest-tag.env b/openshift/integration/koji/pipelines/jobs/mbs-trigger-on-latest-tag.env index af4de5b2..c8dfba3b 100644 --- a/openshift/integration/koji/pipelines/jobs/mbs-trigger-on-latest-tag.env +++ b/openshift/integration/koji/pipelines/jobs/mbs-trigger-on-latest-tag.env @@ -1,4 +1,4 @@ NAME=mbs-trigger-on-latest-tag MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-trigger-on-latest-tag.VirtualTopic.eng.repotracker.container.tag.> -TEST_JOB_NAME=mbs-stage-integration-test +ENVIRONMENT=stage TRACKED_TAG=latest diff --git a/openshift/integration/koji/pipelines/jobs/mbs-trigger-on-stage-tag.env b/openshift/integration/koji/pipelines/jobs/mbs-trigger-on-stage-tag.env index 9d31b6ca..ec81502e 100644 --- a/openshift/integration/koji/pipelines/jobs/mbs-trigger-on-stage-tag.env +++ b/openshift/integration/koji/pipelines/jobs/mbs-trigger-on-stage-tag.env @@ -1,4 +1,4 @@ NAME=mbs-trigger-on-stage-tag MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-trigger-on-stage-tag.VirtualTopic.eng.repotracker.container.tag.> -TEST_JOB_NAME=mbs-prod-integration-test TRACKED_TAG=stage +ENVIRONMENT=prod diff --git a/openshift/integration/koji/pipelines/templates/mbs-build-template.yaml b/openshift/integration/koji/pipelines/templates/mbs-build-template.yaml index 57fca670..4b80f907 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-build-template.yaml +++ b/openshift/integration/koji/pipelines/templates/mbs-build-template.yaml @@ -90,13 +90,6 @@ parameters: - name: MBS_FRONTEND_IMAGESTREAM_NAMESPACE displayName: Namespace of ImageStream for MBS frontend images required: false -- name: MBS_INTEGRATION_TEST_BUILD_CONFIG_NAME - displayName: Name of BuildConfig for running integration tests - required: true - value: mbs-dev-integration-test -- name: MBS_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE - displayName: Namespace of BuildConfig for running integration tests - required: false - name: FORCE_PUBLISH_IMAGE displayName: Whether to push the resulting image regardless of the Git branch value: "false" @@ -153,10 +146,19 @@ parameters: displayName: The lifetime of the OpenShift project allocated by C3I-as-a-Service. required: true value: "120" +- name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE + displayName: The namespace where the Pipeline-as-a-Service project request BuildConfig has been defined + required: false + value: c3i - name: CLEANUP displayName: Cleanup objects after the pipeline is complete required: true value: "true" +- name: ENVIRONMENT + displayName: environment name (dev/stage/prod) + required: true + value: dev +{% include "snippets/c3i-library-parameters.yaml" %} labels: template: mbs-build objects: @@ -223,10 +225,6 @@ objects: spec: runPolicy: "Parallel" completionDeadlineSeconds: 1800 - source: - git: - uri: "${MBS_GIT_REPO}" - ref: "${MBS_GIT_REF}" strategy: type: JenkinsPipeline jenkinsPipelineStrategy: @@ -263,10 +261,6 @@ objects: value: "${MBS_FRONTEND_IMAGESTREAM_NAMESPACE}" - name: MBS_MAIN_BRANCH value: "${MBS_MAIN_BRANCH}" - - name: MBS_INTEGRATION_TEST_BUILD_CONFIG_NAME - value: "${MBS_INTEGRATION_TEST_BUILD_CONFIG_NAME}" - - name: MBS_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE - value: "${MBS_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE}" - name: PAGURE_REPO_NAME value: "${PAGURE_REPO_NAME}" - name: PAGURE_REPO_IS_FORK @@ -285,6 +279,8 @@ objects: value: "${EXTRA_RPMS}" - name: TESTCASES value: "${TESTCASES}" + - name: ENVIRONMENT + value: "${ENVIRONMENT}" - name: USE_C3IAAS value: "${USE_C3IAAS}" - name: C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE @@ -293,6 +289,9 @@ objects: value: "${C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAME}" - name: C3IAAS_LIFETIME value: "${C3IAAS_LIFETIME}" + - name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE + value: "${PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE}" - name: CLEANUP value: "${CLEANUP}" - jenkinsfilePath: openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile + jenkinsfile: | + {% filter indent(width=10) %}{% include "mbs-build.Jenkinsfile" %}{% endfilter %} diff --git a/openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile b/openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile index 00759c76..c6ea1ef1 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile +++ b/openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile @@ -1,43 +1,7 @@ -library identifier: 'c3i@master', changelog: false, - retriever: modernSCM([$class: 'GitSCMSource', remote: 'https://pagure.io/c3i-library.git']) +{% include "snippets/c3i-library.groovy" %} import static org.apache.commons.lang.StringEscapeUtils.escapeHtml; pipeline { - agent { - kubernetes { - cloud params.JENKINS_AGENT_CLOUD_NAME - label "jenkins-slave-${UUID.randomUUID().toString()}" - serviceAccount params.JENKINS_AGENT_SERVICE_ACCOUNT - defaultContainer 'jnlp' - yaml """ - apiVersion: v1 - kind: Pod - metadata: - labels: - app: "jenkins-${env.JOB_BASE_NAME.take(50)}" - factory2-pipeline-kind: "mbs-build-pipeline" - factory2-pipeline-build-number: "${env.BUILD_NUMBER}" - spec: - containers: - - name: jnlp - image: "${params.JENKINS_AGENT_IMAGE}" - imagePullPolicy: Always - tty: true - env: - - name: REGISTRY_CREDENTIALS - valueFrom: - secretKeyRef: - name: "${params.CONTAINER_REGISTRY_CREDENTIALS}" - key: ".dockerconfigjson" - resources: - requests: - memory: 512Mi - cpu: 300m - limits: - memory: 768Mi - cpu: 500m - """ - } - } + {% include "snippets/default-agent.groovy" %} options { timestamps() timeout(time: 120, unit: 'MINUTES') @@ -64,6 +28,7 @@ pipeline { env.GIT_COMMIT = env.MBS_GIT_COMMIT echo "Build ${params.MBS_GIT_REF}, commit=${env.MBS_GIT_COMMIT}" + env.IMAGE_IS_SCRATCH = (params.MBS_GIT_REF != params.MBS_MAIN_BRANCH) // Is the current branch a pull-request? If no, env.PR_NO will be empty. env.PR_NO = getPrNo(params.MBS_GIT_REF) @@ -102,6 +67,7 @@ pipeline { -e '/sitepackages/a setenv = PYTHONPATH={toxinidir}' \ tox.ini """ + {% include "snippets/get_paas_domain.groovy" %} } } } @@ -304,33 +270,7 @@ pipeline { } } } - stage('Run integration tests') { - steps { - script { - openshift.withCluster() { - openshift.withProject(params.MBS_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE) { - def build = c3i.buildAndWait(script: this, objs: "bc/${params.MBS_INTEGRATION_TEST_BUILD_CONFIG_NAME}", - '-e', "MBS_GIT_REPO=${params.MBS_GIT_REPO}", - '-e', "MBS_GIT_REF=${env.PR_NO ? params.MBS_GIT_REF : env.MBS_GIT_COMMIT}", - '-e', "MBS_BACKEND_IMAGE=${env.BACKEND_IMAGE_REF}", - '-e', "MBS_FRONTEND_IMAGE=${env.FRONTEND_IMAGE_REF}", - '-e', "TEST_IMAGES=${env.BACKEND_IMAGE_REF},${env.FRONTEND_IMAGE_REF}", - '-e', "IMAGE_IS_SCRATCH=${params.MBS_GIT_REF != params.MBS_MAIN_BRANCH}", - '-e', "TEST_NAMESPACE=${env.PIPELINE_ID}", - '-e', "TESTCASES='${params.TESTCASES}'", - '-e', "CLEANUP=${params.CLEANUP}" - ) - echo 'Integration tests PASSED' - } - } - } - } - post { - failure { - echo 'Integration tests FAILED' - } - } - } + {% include "snippets/mbs-integration-test.groovy" %} stage('Push images') { when { expression { @@ -340,10 +280,13 @@ pipeline { } steps { script { - if (env.REGISTRY_CREDENTIALS) { - dir ("${env.HOME}/.docker") { - writeFile file: 'config.json', text: env.REGISTRY_CREDENTIALS - } + if (params.CONTAINER_REGISTRY_CREDENTIALS) { + dir ("${env.HOME}/.docker") { + openshift.withCluster() { + def dockerconf = openshift.selector('secret', params.CONTAINER_REGISTRY_CREDENTIALS).object().data['.dockerconfigjson'] + writeFile file: 'config.json', text: dockerconf, encoding: "Base64" + } + } } def registryToken = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/token') def copyDown = { name, src -> @@ -412,13 +355,13 @@ pipeline { steps { script { openshift.withCluster() { - openshift.withProject(params.MBS_BACKEND_IMAGESTREAM_NAMESPACE) { + openshift.withProject(params.MBS_BACKEND_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_ID) { def sourceRef = "${params.MBS_BACKEND_IMAGESTREAM_NAME}@${env.BACKEND_IMAGE_DIGEST}" def destRef = "${params.MBS_BACKEND_IMAGESTREAM_NAME}:${params.MBS_DEV_IMAGE_TAG}" echo "Tagging ${sourceRef} as ${destRef}..." openshift.tag(sourceRef, destRef) } - openshift.withProject(params.MBS_FRONTEND_IMAGESTREAM_NAMESPACE) { + openshift.withProject(params.MBS_FRONTEND_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_ID) { def sourceRef = "${params.MBS_FRONTEND_IMAGESTREAM_NAME}@${env.FRONTEND_IMAGE_DIGEST}" def destRef = "${params.MBS_FRONTEND_IMAGESTREAM_NAME}:${params.MBS_DEV_IMAGE_TAG}" echo "Tagging ${sourceRef} as ${destRef}..." @@ -442,13 +385,13 @@ pipeline { openshift.withProject(env.PIPELINE_ID) { if (env.BACKEND_IMAGE_TAG) { echo "Removing tag ${env.BACKEND_IMAGE_TAG} from the ${params.MBS_BACKEND_IMAGESTREAM_NAME} ImageStream..." - openshift.withProject(params.MBS_BACKEND_IMAGESTREAM_NAMESPACE) { + openshift.withProject(params.MBS_BACKEND_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_ID) { openshift.tag("${params.MBS_BACKEND_IMAGESTREAM_NAME}:${env.BACKEND_IMAGE_TAG}", "-d") } } if (env.FRONTEND_IMAGE_TAG) { echo "Removing tag ${env.FRONTEND_IMAGE_TAG} from the ${params.MBS_FRONTEND_IMAGESTREAM_NAME} ImageStream..." - openshift.withProject(params.MBS_FRONTEND_IMAGESTREAM_NAMESPACE) { + openshift.withProject(params.BS_FRONTEND_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_ID) { openshift.tag("${params.MBS_FRONTEND_IMAGESTREAM_NAME}:${env.FRONTEND_IMAGE_TAG}", "-d") } } @@ -539,3 +482,4 @@ def sendBuildStatusEmail(String status) { } emailext to: recipient, subject: subject, body: body } +{% include "snippets/functions.groovy" %} diff --git a/openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.Jenkinsfile b/openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.Jenkinsfile index 2add6d30..5cf996d5 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.Jenkinsfile +++ b/openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.Jenkinsfile @@ -1,79 +1,122 @@ -// Use scripted syntax because CIBuildTrigger currently doesn't support the declarative syntax -properties([ - buildDiscarder(logRotator(numToKeepStr: '10')), - pipelineTriggers([ - // example: https://github.com/jenkinsci/jms-messaging-plugin/blob/9b9387c3a52f037ba0d019c2ebcf2a2796fc6397/src/test/java/com/redhat/jenkins/plugins/ci/integration/AmqMessagingPluginIntegrationTest.java - [$class: 'CIBuildTrigger', - providerData: [$class: 'ActiveMQSubscriberProviderData', - name: params.MESSAGING_PROVIDER, - overrides: [topic: params.MESSAGING_TOPIC], - checks: [ - [field: '$.msg.subject_type', expectedValue: 'container-image'], - [field: '$.msg.subject_identifier', expectedValue: params.SUBJECT_IDENTIFIER_REGEX], - [field: '$.msg.decision_context', expectedValue: params.DECISION_CONTEXT_REGEX], - [field: '$.msg.policies_satisfied', expectedValue: 'true'], - ], - ], - ], - ]), -]) - -if (!params.CI_MESSAGE) { - echo 'This build is not started by a CI message. Only configurations were done.' - return -} - -def label = "jenkins-slave-${UUID.randomUUID().toString()}" -podTemplate( - cloud: "${params.JENKINS_AGENT_CLOUD_NAME}", - label: label, - serviceAccount: "${params.JENKINS_AGENT_SERVICE_ACCOUNT}", - defaultContainer: 'jnlp', - yaml: """ - apiVersion: v1 - kind: Pod - metadata: - labels: - app: "jenkins-${env.JOB_BASE_NAME.take(50)}" - factory2-pipeline-kind: "mbs-greenwave-trigger" - factory2-pipeline-build-number: "${env.BUILD_NUMBER}" - spec: - containers: - - name: jnlp - image: ${params.JENKINS_AGENT_IMAGE} - imagePullPolicy: Always - tty: true - resources: - requests: - memory: 256Mi - cpu: 200m - limits: - memory: 512Mi - cpu: 300m - """ -) { - node(label) { - stage('Trigger promotion') { - def message = readJSON text: params.CI_MESSAGE - // Extract the digest of the image to be promoted. - // e.g. factory2/waiverdb@sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b - // -> sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b - def digest = (message.msg.subject_identifier =~ /@(sha256:\w+)$/)[0][1] - // Generate the pull spec of the image - // e.g. quay.io/factory2/waiverdb@sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b - def image = "${params.SOURCE_CONTAINER_REPO}@${digest}" - echo "Starting a new build to promote image ${image} to :${params.TARGET_TAG}..." - openshift.withCluster() { - def bcSelector = openshift.selector('bc', params.IMAGE_PROMOTION_JOB) - def buildSelector = bcSelector.startBuild( - '-e', "IMAGE=${image}", - '-e', "DEST_TAG=${params.TARGET_TAG}", - ) - bcSelector.watch { - return !(it.object().status.phase in ["New", "Pending"]) +{% include "snippets/c3i-library.groovy" %} +pipeline { + {% include "snippets/default-agent.groovy" %} + options { + timestamps() + timeout(time: 30, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '10')) + } + environment { + PIPELINE_NAMESPACE = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/namespace').trim() + SERVICE_ACCOUNT_TOKEN = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/token').trim() + } + triggers { + ciBuildTrigger( + noSquash: false, + providerList: [ + activeMQSubscriber( + name: params.MESSAGING_PROVIDER, + overrides: [topic: params.MESSAGING_TOPIC], + checks: [ + [field: '$.msg.subject_type', expectedValue: 'container-image'], + [field: '$.msg.subject_identifier', expectedValue: params.SUBJECT_IDENTIFIER_REGEX], + [field: '$.msg.decision_context', expectedValue: params.DECISION_CONTEXT_REGEX], + [field: '$.msg.policies_satisfied', expectedValue: 'true'], + ] + ) + ] + ) + } + stages { + stage("Message Check and setup") { + steps { + script { + if (!params.CI_MESSAGE) { + error("This build is not started by a CI message. Only configurations were done.") } - buildInfo = buildSelector.object() - echo "Build ${buildInfo.metadata.annotations['openshift.io/jenkins-build-uri'] ?: buildInfo.metadata.name} started." + def message = readJSON text: params.CI_MESSAGE + // Extract the digest of the image to be promoted. + // e.g. factory2/waiverdb@sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b + // -> sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b + def digest = (message.msg.subject_identifier =~ /@(sha256:\w+)$/)[0][1] + // Generate the pull spec of the image + // e.g. quay.io/mkovarik/waiverdb@sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b + env.IMAGE = "${params.SOURCE_CONTAINER_REPO}@${digest}" + echo "Starting promotion of image ${env.IMAGE} to :${params.TARGET_TAG}..." + // Setting up registry credentials + dir ("${env.HOME}/.docker") { + // for the OpenShift internal registry + def dockerConfig = readJSON text: '{ "auths": {} }' + dockerConfig.auths['docker-registry.default.svc:5000'] = [ + 'email': '', + 'auth': sh(returnStdout: true, script: 'set +x; echo -n "serviceaccount:$SERVICE_ACCOUNT_TOKEN" | base64 -').trim() + ] + // merging user specified credentials + if (params.CONTAINER_REGISTRY_CREDENTIALS) { + openshift.withCluster() { + def dockerconf = openshift.selector('secret', params.CONTAINER_REGISTRY_CREDENTIALS).object().data['.dockerconfigjson'] + def dockerString = new String(dockerconf.decodeBase64()) + toBeMerged = readJSON text: dockerString + dockerConfig.auths.putAll(toBeMerged.auths) + } + } + // writing to ~/.docker/config.json + writeJSON file: 'config.json', json: dockerConfig + } + } + } + } + stage('Pull image') { + steps { + echo "Pulling container image ${env.IMAGE}..." + withEnv(["SOURCE_IMAGE_REF=${env.IMAGE}"]) { + sh ''' + set -e +x # hide the token from Jenkins console + mkdir -p _image + skopeo copy docker://"$SOURCE_IMAGE_REF" dir:_image + ''' + } + } + } + stage('Promote image') { + steps { + script { + def destinations = params.PROMOTING_DESTINATIONS ? params.PROMOTING_DESTINATIONS.split(',') : [] + openshift.withCluster() { + def pushTasks = destinations.collectEntries { + ["Pushing ${it}" : { + def dest = "${it}:${params.TARGET_TAG}" + // Only docker and atomic registries are allowed + if (!dest.startsWith('atomic:') && !dest.startsWith('docker://')) { + dest = "docker://${dest}" + } + echo "Pushing container image to ${dest}..." + withEnv(["DEST_IMAGE_REF=${dest}"]) { + retry(5) { + sh 'skopeo copy dir:_image "$DEST_IMAGE_REF"' + } + } + }] + } + parallel pushTasks + } + } + } + } + stage('Tag ImageStream') { + when { + expression { + return params.DEST_IMAGESTREAM_NAME && params.TAG_INTO_IMAGESTREAM == "true" + } + } + steps { + script { + def destRef = "${params.DEST_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_NAMESPACE}/${params.DEST_IMAGESTREAM_NAME}:${params.TARGET_TAG}" + openshift.withCluster() { + echo "Tagging ${env.IMAGE} into ${destRef}..." + openshift.tag('--source=docker', env.IMAGE, destRef) + } + } } } } diff --git a/openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.yaml b/openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.yaml index 0d36c31f..8fc17e8f 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.yaml +++ b/openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.yaml @@ -32,9 +32,18 @@ parameters: - name: TARGET_TAG displayName: Tag name to promote the image to required: true -- name: IMAGE_PROMOTION_JOB - displayName: Downstream image promotion job to trigger +- name: TAG_INTO_IMAGESTREAM + displayName: Whether to tag the image into an ImageStream + value: "false" required: true +- name: DEST_IMAGESTREAM_NAME + displayName: Name of the ImageStream to be tagged + required: false + value: "" +- name: CONTAINER_REGISTRY_CREDENTIALS + displayName: Secret name of container registries used for pulling and pushing images + value: factory2-pipeline-registry-credentials + required: false - name: MESSAGING_PROVIDER displayName: Name of the JMS messaging provider value: Red Hat UMB @@ -47,6 +56,11 @@ parameters: - name: JENKINS_AGENT_CLOUD_NAME displayName: Name of OpenShift cloud in Jenkins master configuration value: openshift +- name: PROMOTING_DESTINATIONS + displayName: Comma seperated list of container repositories (without tags) to which the image will be promoted + description: OpenShift registries must be prefixed with 'atomic:' + required: true +{% include "snippets/c3i-library-parameters.yaml" %} objects: - kind: ServiceAccount apiVersion: v1 @@ -74,14 +88,12 @@ objects: spec: runPolicy: "Parallel" completionDeadlineSeconds: 1800 - source: - git: - uri: "${MBS_GIT_REPO}" - ref: "${MBS_GIT_REF}" strategy: type: JenkinsPipeline jenkinsPipelineStrategy: env: + - name: PROMOTING_DESTINATIONS + value: "${PROMOTING_DESTINATIONS}" - name: JENKINS_AGENT_CLOUD_NAME value: "${JENKINS_AGENT_CLOUD_NAME}" - name: JENKINS_AGENT_IMAGE @@ -90,10 +102,14 @@ objects: value: "${NAME}-jenkins-slave" - name: SOURCE_CONTAINER_REPO value: "${SOURCE_CONTAINER_REPO}" + - name: CONTAINER_REGISTRY_CREDENTIALS + value: "${CONTAINER_REGISTRY_CREDENTIALS}" - name: TARGET_TAG value: "${TARGET_TAG}" - - name: IMAGE_PROMOTION_JOB - value: "${IMAGE_PROMOTION_JOB}" + - name: TAG_INTO_IMAGESTREAM + value: "${TAG_INTO_IMAGESTREAM}" + - name: DEST_IMAGESTREAM_NAME + value: "${DEST_IMAGESTREAM_NAME}" - name: DECISION_CONTEXT_REGEX value: "${DECISION_CONTEXT_REGEX}" - name: SUBJECT_IDENTIFIER_REGEX @@ -107,4 +123,5 @@ objects: value: - name: MESSAGE_HEADERS value: - jenkinsfilePath: openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.Jenkinsfile + jenkinsfile: | + {% filter indent(width=10) %}{% include "mbs-greenwave-trigger.Jenkinsfile" %}{% endfilter %} diff --git a/openshift/integration/koji/pipelines/templates/mbs-image-promotion-template.yaml b/openshift/integration/koji/pipelines/templates/mbs-image-promotion-template.yaml deleted file mode 100644 index 4a38ab5e..00000000 --- a/openshift/integration/koji/pipelines/templates/mbs-image-promotion-template.yaml +++ /dev/null @@ -1,120 +0,0 @@ -# Template to produce a pipeline for promoting images between environments -# -# The pipeline pulls the image to be promoted, then pushes it to destinations with promoted tags. -# Optionally, it tags the promoted image into an image stream after pushes. ---- -apiVersion: v1 -kind: Template -metadata: - name: mbs-image-promotion -labels: - template: mbs-image-promotion -parameters: -- name: NAME - displayName: Short unique identifier for the templated instances - description: This field is used to deploy multiple pipelines to one OpenShift project from this template. - required: true -- name: MBS_GIT_REPO - displayName: MBS Git repo URL - description: Default MBS Git repo URL in which to run functional tests against - required: true - value: "https://pagure.io/fm-orchestrator.git" -- name: MBS_GIT_REF - displayName: MBS Git repo ref - description: Default MBS Git repo ref in which to run functional tests against - required: true - value: master -- name: IMAGE - displayName: The container image to be promoted - description: This field must be in repo:tag or repo@sha256 format - value: provided-by-trigger -- name: PROMOTING_DESTINATIONS - displayName: Comma seperated list of container repositories (without tags) to which the image will be promoted - description: OpenShift registries must be prefixed with 'atomic:' - required: true -- name: CONTAINER_REGISTRY_CREDENTIALS - displayName: Secret name of container registries used for pulling and pushing images - value: factory2-pipeline-registry-credentials - required: false -- name: TAG_INTO_IMAGESTREAM - displayName: Whether to tag the image into an ImageStream - value: "false" - required: true -- name: DEST_IMAGESTREAM_NAME - displayName: Name of the ImageStream to be tagged - required: false - value: "" -- name: DEST_IMAGESTREAM_NAMESPACE - displayName: Namespace of the ImageStream to be tagged - description: Leaving blank means using the same namespace as the pipeline build - required: false - value: "" -- name: DEST_TAG - displayName: Name of the new tag - required: true -- name: JENKINS_AGENT_IMAGE - displayName: Container image for Jenkins slave pods - required: true - value: quay.io/factory2/mbs-jenkins-slave:latest -- name: JENKINS_AGENT_CLOUD_NAME - displayName: Name of OpenShift cloud in Jenkins master configuration - required: true - value: openshift -objects: -- kind: ServiceAccount - apiVersion: v1 - metadata: - name: "${NAME}-jenkins-slave" - labels: - app: "${NAME}" -- kind: RoleBinding - apiVersion: v1 - metadata: - name: "${NAME}-jenkins-slave_edit" - labels: - app: "${NAME}" - subjects: - - kind: ServiceAccount - name: "${NAME}-jenkins-slave" - roleRef: - name: edit -- kind: "BuildConfig" - apiVersion: "v1" - metadata: - name: "${NAME}" - labels: - app: "${NAME}" - spec: - runPolicy: "Parallel" - completionDeadlineSeconds: 1800 - source: - git: - uri: "${MBS_GIT_REPO}" - ref: "${MBS_GIT_REF}" - strategy: - type: JenkinsPipeline - source: - type: None - jenkinsPipelineStrategy: - env: - - name: IMAGE - value: "${IMAGE}" - - name: PROMOTING_DESTINATIONS - value: "${PROMOTING_DESTINATIONS}" - - name: CONTAINER_REGISTRY_CREDENTIALS - value: "${CONTAINER_REGISTRY_CREDENTIALS}" - - name: TAG_INTO_IMAGESTREAM - value: "${TAG_INTO_IMAGESTREAM}" - - name: DEST_IMAGESTREAM_NAME - value: "${DEST_IMAGESTREAM_NAME}" - - name: DEST_IMAGESTREAM_NAMESPACE - value: "${DEST_IMAGESTREAM_NAMESPACE}" - - name: DEST_TAG - value: "${DEST_TAG}" - - name: JENKINS_AGENT_IMAGE - value: "${JENKINS_AGENT_IMAGE}" - - name: JENKINS_AGENT_CLOUD_NAME - value: "${JENKINS_AGENT_CLOUD_NAME}" - - name: JENKINS_AGENT_SERVICE_ACCOUNT - value: "${NAME}-jenkins-slave" - jenkinsfilePath: openshift/integration/koji/pipelines/templates/mbs-image-promotion.Jenkinsfile diff --git a/openshift/integration/koji/pipelines/templates/mbs-image-promotion.Jenkinsfile b/openshift/integration/koji/pipelines/templates/mbs-image-promotion.Jenkinsfile deleted file mode 100644 index c4e37857..00000000 --- a/openshift/integration/koji/pipelines/templates/mbs-image-promotion.Jenkinsfile +++ /dev/null @@ -1,125 +0,0 @@ -pipeline { - agent { - kubernetes { - cloud "${params.JENKINS_AGENT_CLOUD_NAME}" - label "jenkins-slave-${UUID.randomUUID().toString()}" - serviceAccount "${params.JENKINS_AGENT_SERVICE_ACCOUNT}" - defaultContainer 'jnlp' - yaml """ - apiVersion: v1 - kind: Pod - metadata: - labels: - app: "jenkins-${env.JOB_BASE_NAME.take(50)}" - factory2-pipeline-kind: "mbs-image-promotion-pipeline" - factory2-pipeline-build-number: "${env.BUILD_NUMBER}" - spec: - containers: - - name: jnlp - image: "${params.JENKINS_AGENT_IMAGE}" - imagePullPolicy: Always - tty: true - env: - - name: REGISTRY_CREDENTIALS - valueFrom: - secretKeyRef: - name: "${params.CONTAINER_REGISTRY_CREDENTIALS}" - key: '.dockerconfigjson' - resources: - requests: - memory: 512Mi - cpu: 300m - limits: - memory: 768Mi - cpu: 500m - """ - } - } - options { - timestamps() - timeout(time: 30, unit: 'MINUTES') - buildDiscarder(logRotator(numToKeepStr: '10')) - skipDefaultCheckout() - } - environment { - PIPELINE_NAMESPACE = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/namespace').trim() - SERVICE_ACCOUNT_TOKEN = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/token').trim() - } - stages { - stage ('Prepare') { - steps { - script { - // Setting up registry credentials - dir ("${env.HOME}/.docker") { - // for the OpenShift internal registry - def dockerConfig = readJSON text: '{ "auths": {} }' - dockerConfig.auths['docker-registry.default.svc:5000'] = [ - 'email': '', - 'auth': sh(returnStdout: true, script: 'set +x; echo -n "serviceaccount:$SERVICE_ACCOUNT_TOKEN" | base64 -').trim() - ] - // merging user specified credentials - if (env.REGISTRY_CREDENTIALS) { - toBeMerged = readJSON text: env.REGISTRY_CREDENTIALS - dockerConfig.auths.putAll(toBeMerged.auths) - } - // writing to ~/.docker/config.json - writeJSON file: 'config.json', json: dockerConfig - } - } - } - } - stage('Pull image') { - steps { - echo "Pulling container image ${params.IMAGE}..." - withEnv(["SOURCE_IMAGE_REF=${params.IMAGE}"]) { - sh ''' - set -e +x # hide the token from Jenkins console - mkdir -p _image - skopeo copy docker://"$SOURCE_IMAGE_REF" dir:_image - ''' - } - } - } - stage('Promote image') { - steps { - script { - def destinations = params.PROMOTING_DESTINATIONS ? params.PROMOTING_DESTINATIONS.split(',') : [] - openshift.withCluster() { - def pushTasks = destinations.collectEntries { - ["Pushing ${it}" : { - def dest = "${it}:${params.DEST_TAG}" - // Only docker and atomic registries are allowed - if (!dest.startsWith('atomic:') && !dest.startsWith('docker://')) { - dest = "docker://${dest}" - } - echo "Pushing container image to ${dest}..." - withEnv(["DEST_IMAGE_REF=${dest}"]) { - retry(5) { - sh 'skopeo copy dir:_image "$DEST_IMAGE_REF"' - } - } - }] - } - parallel pushTasks - } - } - } - } - stage('Tag ImageStream') { - when { - expression { - return params.DEST_IMAGESTREAM_NAME && params.TAG_INTO_IMAGESTREAM == "true" - } - } - steps { - script { - def destRef = "${params.DEST_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_NAMESPACE}/${params.DEST_IMAGESTREAM_NAME}:${params.DEST_TAG}" - openshift.withCluster() { - echo "Tagging ${params.IMAGE} into ${destRef}..." - openshift.tag('--source=docker', params.IMAGE, destRef) - } - } - } - } - } -} diff --git a/openshift/integration/koji/pipelines/templates/mbs-integration-test-template.yaml b/openshift/integration/koji/pipelines/templates/mbs-integration-test-template.yaml deleted file mode 100644 index 337554e2..00000000 --- a/openshift/integration/koji/pipelines/templates/mbs-integration-test-template.yaml +++ /dev/null @@ -1,143 +0,0 @@ -# Template to produce a new OpenShift pipeline for running integration tests -# ---- -apiVersion: v1 -kind: Template -metadata: - name: mbs-integration-test -labels: - template: mbs-integration-test -parameters: -- name: NAME - displayName: Short unique identifier for the templated instances - description: This field is used to deploy multiple pipelines to one OpenShift project from this template. - required: true - value: mbs-integration-test -- name: MBS_BACKEND_IMAGE - displayName: The MBS backend container image to be tested - description: This field must be in repo:tag or repo@sha256 format - value: quay.io/factory2/mbs-backend:latest -- name: MBS_FRONTEND_IMAGE - displayName: The MBS frontend container image to be tested - description: This field must be in repo:tag or repo@sha256 format - value: quay.io/factory2/mbs-frontend:latest -- name: MBS_GIT_REPO - displayName: MBS Git repo URL - description: Default MBS Git repo URL in which to find the integration tests to run - required: true - value: "https://pagure.io/fm-orchestrator.git" -- name: MBS_GIT_REF - displayName: MBS Git repo ref - description: Default MBS Git repo ref in which to find the integration tests to run - required: true - value: master -- name: KOJI_IMAGE - displayName: The Koji container image to be tested - description: This field must be in repo:tag or repo@sha256 format - value: quay.io/factory2/koji:latest -- name: TEST_IMAGES - displayName: Images being tested - description: >- - A space-separated list of the refs of the images being tested. - Results of the tests will be reported to ResultsDB. -- name: JENKINS_AGENT_IMAGE - displayName: Container image for Jenkins slave pods - required: true - value: quay.io/factory2/mbs-jenkins-slave:latest -- name: JENKINS_AGENT_CLOUD_NAME - displayName: Name of OpenShift cloud in Jenkins master configuration - required: true - value: openshift -- name: TEST_NAMESPACE - displayName: >- - Namespace where the Jenkins agent for this test will run, and where - test resources will be created. - required: false -- name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE - displayName: Namespace with pipeline-as-a-service build - value: c3i -- name: ENVIRONMENT - displayName: environment name (dev/stage/prod) - required: true - value: dev -- name: MESSAGING_PROVIDER - displayName: Name of the JMS messaging provider - value: Red Hat UMB -- name: TESTCASES - displayName: >- - Space-separated list of testcases to run as part of the pipeline. An empty string (the default) - causes all available testcases to run. The value "skip" causes no testcases to be run. - required: false - value: "" -- name: CLEANUP - displayName: Cleanup objects after testing is complete - required: true - value: "true" -objects: -- kind: ServiceAccount - apiVersion: v1 - metadata: - name: "${NAME}-jenkins-slave" - labels: - app: "${NAME}" -- kind: RoleBinding - apiVersion: v1 - metadata: - name: "${NAME}-jenkins-slave_edit" - labels: - app: "${NAME}" - subjects: - - kind: ServiceAccount - name: "${NAME}-jenkins-slave" - roleRef: - name: edit -- kind: "BuildConfig" - apiVersion: "v1" - metadata: - name: "${NAME}" - labels: - app: "${NAME}" - spec: - runPolicy: "Parallel" - completionDeadlineSeconds: 1800 - source: - git: - uri: "${MBS_GIT_REPO}" - ref: "${MBS_GIT_REF}" - strategy: - type: JenkinsPipeline - jenkinsPipelineStrategy: - env: - - name: MBS_GIT_REPO - value: "${MBS_GIT_REPO}" - - name: MBS_GIT_REF - value: "${MBS_GIT_REF}" - - name: MBS_BACKEND_IMAGE - value: "${MBS_BACKEND_IMAGE}" - - name: MBS_FRONTEND_IMAGE - value: "${MBS_FRONTEND_IMAGE}" - - name: KOJI_IMAGE - value: "${KOJI_IMAGE}" - - name: TEST_IMAGES - value: "${TEST_IMAGES}" - - name: IMAGE_IS_SCRATCH - value: "true" - - name: JENKINS_AGENT_IMAGE - value: "${JENKINS_AGENT_IMAGE}" - - name: JENKINS_AGENT_CLOUD_NAME - value: "${JENKINS_AGENT_CLOUD_NAME}" - - name: TEST_NAMESPACE - value: "${TEST_NAMESPACE}" - - name: ENVIRONMENT - value: "${ENVIRONMENT}" - - name: MESSAGING_PROVIDER - value: "${MESSAGING_PROVIDER}" - - name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE - value: "${PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE}" - - name: JENKINS_AGENT_SERVICE_ACCOUNT - value: "${NAME}-jenkins-slave" - - name: TESTCASES - value: "${TESTCASES}" - - name: CLEANUP - value: "${CLEANUP}" - jenkinsfilePath: openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile diff --git a/openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile b/openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile deleted file mode 100644 index 5ba423ea..00000000 --- a/openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile +++ /dev/null @@ -1,267 +0,0 @@ -library identifier: 'c3i@master', changelog: false, - retriever: modernSCM([$class: 'GitSCMSource', remote: 'https://pagure.io/c3i-library.git']) -def deployments -pipeline { - agent { - kubernetes { - cloud params.JENKINS_AGENT_CLOUD_NAME - label "jenkins-slave-${UUID.randomUUID().toString()}" - serviceAccount params.JENKINS_AGENT_SERVICE_ACCOUNT - defaultContainer 'jnlp' - yaml """ - apiVersion: v1 - kind: Pod - metadata: - labels: - app: "jenkins-${env.JOB_BASE_NAME.take(50)}" - factory2-pipeline-kind: "mbs-integration-test-pipeline" - factory2-pipeline-build-number: "${env.BUILD_NUMBER}" - spec: - containers: - - name: jnlp - image: "${params.JENKINS_AGENT_IMAGE}" - imagePullPolicy: Always - tty: true - resources: - requests: - memory: 512Mi - cpu: 300m - limits: - memory: 768Mi - cpu: 500m - """ - } - } - options { - timestamps() - timeout(time: 60, unit: 'MINUTES') - buildDiscarder(logRotator(numToKeepStr: '10')) - skipDefaultCheckout() - } - environment { - PIPELINE_ID = "${params.TEST_NAMESPACE}" - } - stages { - stage('Prepare') { - steps { - script { - // MBS_GIT_REF can be either a regular branch (in the heads/ namespace), a pull request - // branch (in the pull/ namespace), or a full 40-character sha1, which is assumed to - // exist on the master branch. - def branch = params.MBS_GIT_REF ==~ '[0-9a-f]{40}' ? 'master' : params.MBS_GIT_REF - c3i.clone(repo: params.MBS_GIT_REPO, branch: branch, rev: params.MBS_GIT_REF) - // get current commit ID - // FIXME: Due to a bug discribed in https://issues.jenkins-ci.org/browse/JENKINS-45489, - // the return value of checkout() is unreliable. - // Not working: env.MBS_GIT_COMMIT = scmVars.GIT_COMMIT - env.MBS_GIT_COMMIT = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() - echo "Running integration tests for ${branch}, commit=${env.MBS_GIT_COMMIT}" - currentBuild.displayName = "${branch}: ${env.MBS_GIT_COMMIT.take(7)}" - } - } - } - stage('Cleanup') { - when { - expression { - // Only run cleanup if we're running tests in the default namespace. - return !params.TEST_NAMESPACE - } - } - steps { - script { - openshift.withCluster() { - openshift.withProject() { - // Cleanup all test environments that were created 1 hour ago in case of failures of previous cleanups. - c3i.cleanup(script: this, 'krb5', 'umb', 'koji', 'mbs') - } - } - } - } - post { - failure { - echo "Cleanup of old environments FAILED" - } - } - } - stage('Route suffix') { - when { - expression { !env.PAAS_DOMAIN } - } - steps { - script { - openshift.withCluster() { - openshift.withProject(env.PIPELINE_ID) { - def testroute = openshift.create('route', 'edge', 'test', '--service=test', '--port=8080') - def testhost = testroute.object().spec.host - env.PAAS_DOMAIN = testhost.minus("test-${env.PIPELINE_ID}.") - testroute.delete() - } - } - } - } - post { - success { - echo "Routes end with ${env.PAAS_DOMAIN}" - } - } - } - stage('Deploy test environment') { - steps { - script { - openshift.withCluster() { - openshift.withProject(params.PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE) { - c3i.buildAndWait(script: this, objs: "bc/pipeline-as-a-service", - '-e', "DEFAULT_IMAGE_TAG=${env.ENVIRONMENT}", - '-e', "PIPELINE_ID=${env.PIPELINE_ID}", - '-e', "WAIVERDB_IMAGE=", - '-e', "C3IAAS_PROJECT=", - '-e', "RESULTSDB_IMAGE=", - '-e', "RESULTSDB_UPDATER_IMAGE=", - '-e', "GREENWAVE_IMAGE=", - '-e', "DATAGREPPER_IMAGE=", - '-e', "DATANOMMER_IMAGE=", - '-e', "MBS_BACKEND_IMAGE=${env.MBS_BACKEND_IMAGE}", - '-e', "MBS_FRONTEND_IMAGE=${env.MBS_FRONTEND_IMAGE}", - '-e', "PAAS_DOMAIN=${env.PAAS_DOMAIN}" - ) - } - } - } - } - } - stage('Run tests') { - steps { - script { - def testcases - if (params.TESTCASES) { - if (params.TESTCASES == 'skip') { - testcases = [] - echo 'Skipping integration tests' - } else { - testcases = params.TESTCASES.split() - echo "Using specified list of testcases: ${testcases}" - } - } else { - testcases = findFiles(glob: 'openshift/integration/koji/pipelines/tests/*.groovy').collect { - it.name.minus('.groovy') - } - echo "Using all available testcases: ${testcases}" - } - testcases.each { testcase -> - env.CURRENT_TESTCASE = testcase - echo "Running testcase ${testcase}..." - def test = load "openshift/integration/koji/pipelines/tests/${testcase}.groovy" - test.runTests() - } - } - } - post { - success { - echo "All tests successful" - } - failure { - echo "Testcase ${env.CURRENT_TESTCASE} FAILED" - } - } - } - } - post { - success { - script { - params.TEST_IMAGES.split(',').each { - sendToResultsDB(it, 'passed') - } - } - } - failure { - script { - params.TEST_IMAGES.split(',').each { - sendToResultsDB(it, 'failed') - } - openshift.withCluster() { - openshift.withProject(params.TEST_NAMESPACE) { - echo 'Getting logs from all deployments...' - openshift.selector('pods', ['c3i.redhat.com/pipeline': env.PIPELINE_ID]).logs('--tail 100') - } - } - } - } - cleanup { - script { - if (params.CLEANUP == 'true' && !params.TEST_NAMESPACE) { - openshift.withCluster() { - /* Tear down everything we just created */ - echo 'Tearing down test resources...' - openshift.selector('all,pvc,configmap,secret', - ['c3i.redhat.com/pipeline': env.PIPELINE_ID]).delete('--ignore-not-found=true') - } - } else { - echo 'Skipping cleanup' - } - } - } - } -} -def sendToResultsDB(imageRef, status) { - if (!params.MESSAGING_PROVIDER) { - echo "Message bus is not set. Skipping send of:\nimageRef: ${imageRef}\nstatus: ${status}" - return - } - def (repourl, digest) = imageRef.tokenize('@') - def (registry, reponame) = repourl.split('/', 2) - def image = reponame.split('/').last() - def sendResult = sendCIMessage \ - providerName: params.MESSAGING_PROVIDER, \ - overrides: [topic: 'VirtualTopic.eng.ci.container-image.test.complete'], \ - messageType: 'Custom', \ - messageProperties: '', \ - messageContent: """ - { - "ci": { - "name": "C3I Jenkins", - "team": "DevOps", - "url": "${env.JENKINS_URL}", - "docs": "https://pagure.io/fm-orchestrator/blob/master/f/openshift/integration/koji", - "irc": "#pnt-devops-dev", - "email": "pnt-factory2-devel@redhat.com", - "environment": "${params.ENVIRONMENT}" - }, - "run": { - "url": "${env.BUILD_URL}", - "log": "${env.BUILD_URL}/console", - "debug": "", - "rebuild": "${env.BUILD_URL}/rebuild/parametrized" - }, - "artifact": { - "type": "container-image", - "repository": "${reponame}", - "digest": "${digest}", - "nvr": "${imageRef}", - "issuer": "c3i-jenkins", - "scratch": ${params.IMAGE_IS_SCRATCH}, - "id": "${image}@${digest}" - }, - "system": - [{ - "os": "${params.JENKINS_AGENT_IMAGE}", - "provider": "openshift", - "architecture": "x86_64" - }], - "type": "integration", - "category": "${params.ENVIRONMENT}", - "status": "${status}", - "xunit": "", - "generated_at": "${new Date().format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone('UTC'))}", - "namespace": "c3i", - "version": "0.1.0" - } - """ - if (sendResult.getMessageId()) { - // echo sent message id and content - echo 'Successfully sent the test result to ResultsDB.' - echo "Message ID: ${sendResult.getMessageId()}" - echo "Message content: ${sendResult.getMessageContent()}" - } else { - echo 'Failed to sent the test result to ResultsDB.' - } -} diff --git a/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.Jenkinsfile b/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.Jenkinsfile new file mode 100644 index 00000000..8b9766d4 --- /dev/null +++ b/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.Jenkinsfile @@ -0,0 +1,103 @@ +{% include "snippets/c3i-library.groovy" %} +pipeline { + {% include "snippets/default-agent.groovy" %} + options { + timestamps() + timeout(time: 60, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '10')) + } + environment { + PIPELINE_NAMESPACE = readFile('/run/secrets/kubernetes.io/serviceaccount/namespace').trim() + PAGURE_URL = "${PAGURE_URL}" + PAGURE_API = "${env.PAGURE_URL}/api/0" + PAGURE_REPO_NAME = "${PAGURE_REPO_NAME}" + PAGURE_REPO_IS_FORK = "${PAGURE_REPO_IS_FORK}" + PAGURE_POLLING_FOR_PR = "${PAGURE_POLLING_FOR_PR}" + PAGURE_REPO_HOME = "${env.PAGURE_URL}${env.PAGURE_REPO_IS_FORK == 'true' ? '/fork' : ''}/${env.PAGURE_REPO_NAME}" + GIT_URL = "${env.PAGURE_URL}/${env.PAGURE_REPO_IS_FORK == 'true' ? 'forks/' : ''}${env.PAGURE_REPO_NAME}.git" + PREMERGE_JOB_NAME = "${PREMERGE_JOB_NAME}" + POSTMERGE_JOB_NAME = "${POSTMERGE_JOB_NAME}" + } + triggers { pollSCM("${PAGURE_POLLING_SCHEDULE}") } + stages { + stage('Prepare') { + agent { label 'master' } + steps { + script { + def polled = env.PAGURE_POLLING_FOR_PR == 'true' ? 'pull/*/head' : "${PAGURE_POLLED_BRANCH}" + // Need to prefix the rev with origin/ for pollSCM to work correctly + def rev = "origin/${polled}" + def scmVars = c3i.clone(repo: env.GIT_URL, branch: polled, rev: rev) + env.GIT_COMMIT = scmVars.GIT_COMMIT + // setting build display name + def prefix = 'origin/' + def branch = scmVars.GIT_BRANCH.startsWith(prefix) ? scmVars.GIT_BRANCH.substring(prefix.size()) + : scmVars.GIT_BRANCH // origin/pull/1234/head -> pull/1234/head, origin/master -> master + env.MBS_GIT_BRANCH = branch + echo "Build on branch=${env.MBS_GIT_BRANCH}, commit=${env.GIT_COMMIT}" + if (env.PAGURE_POLLING_FOR_PR == 'false') { + currentBuild.displayName = "${env.MBS_GIT_BRANCH}: ${env.GIT_COMMIT.substring(0, 7)}" + currentBuild.description = """${currentBuild.displayName}""" + } else if (env.PAGURE_POLLING_FOR_PR == 'true' && branch ==~ /^pull\/[0-9]+\/head$/) { + env.PR_NO = branch.split('/')[1] + def prInfo = pagure.getPR(env.PR_NO) + if (prInfo.status == 'Open') { + env.PR_URL = "${env.PAGURE_REPO_HOME}/pull-request/${env.PR_NO}" + // To HTML syntax in build description, go to `Jenkins/Global Security/Markup Formatter` and select 'Safe HTML'. + def pagureLink = """PR#${env.PR_NO}""" + echo "Building PR #${env.PR_NO}: ${env.PR_URL}" + currentBuild.displayName = "PR#${env.PR_NO}" + currentBuild.description = pagureLink + } else { + echo "Skipping PR#${env.PR_NO} because it is ${prInfo.status}" + env.SKIP = 'true' + } + } else { // This shouldn't happen. + error("Build is aborted due to unexpected polling trigger actions.") + } + } + } + } + stage('Update pipeline jobs') { + when { + expression { + return "${PIPELINE_UPDATE_JOBS_DIR}" && env.PAGURE_POLLING_FOR_PR == 'false' && env.MBS_GIT_BRANCH == "${PAGURE_POLLED_BRANCH}" + } + } + steps { + script { + c3i.clone(repo: env.GIT_URL, branch: env.MBS_GIT_BRANCH) + dir('openshift/integration/koji/pipelines') { + sh ''' + make install JOBS_DIR="${PIPELINE_UPDATE_JOBS_DIR}" + ''' + } + } + } + } + stage('Build') { + when { + not { + environment name: 'SKIP', value: 'true' + } + } + steps { + script { + openshift.withCluster() { + echo 'Starting a MBS build run...' + def devBuild = c3i.build(script: this, + objs: "bc/${env.PAGURE_POLLING_FOR_PR == 'true' ? env.PREMERGE_JOB_NAME : env.POSTMERGE_JOB_NAME}", + '-e', "MBS_GIT_REF=${env.MBS_GIT_BRANCH}", '-e', "PAGURE_REPO_IS_FORK=${env.PAGURE_REPO_IS_FORK}", + '-e', "PAGURE_REPO_NAME=${env.PAGURE_REPO_NAME}" + ) + c3i.waitForBuildStart(script: this, build: devBuild) + def devBuildInfo = devBuild.object() + def downstreamBuildName = devBuildInfo.metadata.name + def downstreamBuildUrl = devBuildInfo.metadata.annotations['openshift.io/jenkins-build-uri'] + echo "Downstream build ${downstreamBuildName}(${downstreamBuildUrl}) started." + } + } + } + } + } +} diff --git a/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml b/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml index 2de9b369..d9aca2c5 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml +++ b/openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml @@ -58,6 +58,7 @@ parameters: displayName: Name of OpenShift cloud in Jenkins master configuration required: true value: openshift +{% include "snippets/c3i-library-parameters.yaml" %} objects: - kind: ServiceAccount apiVersion: v1 @@ -88,139 +89,12 @@ objects: strategy: type: JenkinsPipeline jenkinsPipelineStrategy: + env: + - name: JENKINS_AGENT_CLOUD_NAME + value: "${JENKINS_AGENT_CLOUD_NAME}" + - name: JENKINS_AGENT_IMAGE + value: "${JENKINS_AGENT_IMAGE}" + - name: JENKINS_AGENT_SERVICE_ACCOUNT + value: "${NAME}-jenkins-slave" jenkinsfile: |- - // Don't use external Jenkinsfile here, or Jenkins will also poll on that repo and branch - library identifier: 'c3i@master', changelog: false, - retriever: modernSCM([$class: 'GitSCMSource', remote: 'https://pagure.io/c3i-library.git']) - pipeline { - agent { - kubernetes { - cloud "${JENKINS_AGENT_CLOUD_NAME}" - label "jenkins-slave-${UUID.randomUUID().toString()}" - serviceAccount "${NAME}-jenkins-slave" - defaultContainer 'jnlp' - yaml """ - apiVersion: v1 - kind: Pod - metadata: - labels: - app: "jenkins-${env.JOB_BASE_NAME.take(50)}" - factory2-pipeline-kind: "mbs-polling-to-pagure-pipeline" - factory2-pipeline-build-number: "${env.BUILD_NUMBER}" - spec: - containers: - - name: jnlp - image: "${JENKINS_AGENT_IMAGE}" - imagePullPolicy: Always - tty: true - resources: - requests: - memory: 512Mi - cpu: 300m - limits: - memory: 768Mi - cpu: 500m - """ - } - } - options { - timestamps() - timeout(time: 60, unit: 'MINUTES') - buildDiscarder(logRotator(numToKeepStr: '10')) - skipDefaultCheckout() - } - environment { - PIPELINE_NAMESPACE = readFile('/run/secrets/kubernetes.io/serviceaccount/namespace').trim() - PAGURE_URL = "${PAGURE_URL}" - PAGURE_API = "${env.PAGURE_URL}/api/0" - PAGURE_REPO_NAME = "${PAGURE_REPO_NAME}" - PAGURE_REPO_IS_FORK = "${PAGURE_REPO_IS_FORK}" - PAGURE_POLLING_FOR_PR = "${PAGURE_POLLING_FOR_PR}" - PAGURE_REPO_HOME = "${env.PAGURE_URL}${env.PAGURE_REPO_IS_FORK == 'true' ? '/fork' : ''}/${env.PAGURE_REPO_NAME}" - GIT_URL = "${env.PAGURE_URL}/${env.PAGURE_REPO_IS_FORK == 'true' ? 'forks/' : ''}${env.PAGURE_REPO_NAME}.git" - PREMERGE_JOB_NAME = "${PREMERGE_JOB_NAME}" - POSTMERGE_JOB_NAME = "${POSTMERGE_JOB_NAME}" - } - triggers { pollSCM("${PAGURE_POLLING_SCHEDULE}") } - stages { - stage('Prepare') { - agent { label 'master' } - steps { - script { - def polled = env.PAGURE_POLLING_FOR_PR == 'true' ? 'pull/*/head' : "${PAGURE_POLLED_BRANCH}" - // Need to prefix the rev with origin/ for pollSCM to work correctly - def rev = "origin/${polled}" - def scmVars = c3i.clone(repo: env.GIT_URL, branch: polled, rev: rev) - env.GIT_COMMIT = scmVars.GIT_COMMIT - // setting build display name - def prefix = 'origin/' - def branch = scmVars.GIT_BRANCH.startsWith(prefix) ? scmVars.GIT_BRANCH.substring(prefix.size()) - : scmVars.GIT_BRANCH // origin/pull/1234/head -> pull/1234/head, origin/master -> master - env.MBS_GIT_BRANCH = branch - echo "Build on branch=${env.MBS_GIT_BRANCH}, commit=${env.GIT_COMMIT}" - if (env.PAGURE_POLLING_FOR_PR == 'false') { - currentBuild.displayName = "${env.MBS_GIT_BRANCH}: ${env.GIT_COMMIT.substring(0, 7)}" - currentBuild.description = """${currentBuild.displayName}""" - } else if (env.PAGURE_POLLING_FOR_PR == 'true' && branch ==~ /^pull\/[0-9]+\/head$/) { - env.PR_NO = branch.split('/')[1] - def prInfo = pagure.getPR(env.PR_NO) - if (prInfo.status == 'Open') { - env.PR_URL = "${env.PAGURE_REPO_HOME}/pull-request/${env.PR_NO}" - // To HTML syntax in build description, go to `Jenkins/Global Security/Markup Formatter` and select 'Safe HTML'. - def pagureLink = """PR#${env.PR_NO}""" - echo "Building PR #${env.PR_NO}: ${env.PR_URL}" - currentBuild.displayName = "PR#${env.PR_NO}" - currentBuild.description = pagureLink - } else { - echo "Skipping PR#${env.PR_NO} because it is ${prInfo.status}" - env.SKIP = 'true' - } - } else { // This shouldn't happen. - error("Build is aborted due to unexpected polling trigger actions.") - } - } - } - } - stage('Update pipeline jobs') { - when { - expression { - return "${PIPELINE_UPDATE_JOBS_DIR}" && env.PAGURE_POLLING_FOR_PR == 'false' && env.MBS_GIT_BRANCH == "${PAGURE_POLLED_BRANCH}" - } - } - steps { - script { - c3i.clone(repo: env.GIT_URL, branch: env.MBS_GIT_BRANCH) - dir('openshift/integration/koji/pipelines') { - sh ''' - make install JOBS_DIR="${PIPELINE_UPDATE_JOBS_DIR}" - ''' - } - } - } - } - stage('Build') { - when { - not { - environment name: 'SKIP', value: 'true' - } - } - steps { - script { - openshift.withCluster() { - echo 'Starting a MBS build run...' - def devBuild = c3i.build(script: this, - objs: "bc/${env.PAGURE_POLLING_FOR_PR == 'true' ? env.PREMERGE_JOB_NAME : env.POSTMERGE_JOB_NAME}", - '-e', "MBS_GIT_REF=${env.MBS_GIT_BRANCH}", '-e', "PAGURE_REPO_IS_FORK=${env.PAGURE_REPO_IS_FORK}", - '-e', "PAGURE_REPO_NAME=${env.PAGURE_REPO_NAME}" - ) - c3i.waitForBuildStart(script: this, build: devBuild) - def devBuildInfo = devBuild.object() - def downstreamBuildName = devBuildInfo.metadata.name - def downstreamBuildUrl = devBuildInfo.metadata.annotations['openshift.io/jenkins-build-uri'] - echo "Downstream build ${downstreamBuildName}(${downstreamBuildUrl}) started." - } - } - } - } - } - } + {% filter indent(width=10) %}{% include "mbs-polling-pagure.Jenkinsfile" %}{% endfilter %} diff --git a/openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.Jenkinsfile b/openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.Jenkinsfile index eaa4b718..001760de 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.Jenkinsfile +++ b/openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.Jenkinsfile @@ -1,110 +1,56 @@ -// Use scripted syntax because CIBuildTrigger currently doesn't support the declarative syntax -properties([ - buildDiscarder(logRotator(numToKeepStr: '10')), - pipelineTriggers([ - // example: https://github.com/jenkinsci/jms-messaging-plugin/blob/9b9387c3a52f037ba0d019c2ebcf2a2796fc6397/src/test/java/com/redhat/jenkins/plugins/ci/integration/AmqMessagingPluginIntegrationTest.java - [$class: 'CIBuildTrigger', - providerData: [$class: 'ActiveMQSubscriberProviderData', - name: params.MESSAGING_PROVIDER, - overrides: [topic: params.MESSAGING_TOPIC], - selector: "repo = '${params.TRACKED_CONTAINER_REPO}' AND action IN ('added', 'updated') AND tag = '${params.TRACKED_TAG}'", - ], - ], - ]), -]) - -if (!params.CI_MESSAGE) { - echo 'This build is not started by a CI message. Only configurations were done.' - return -} -library identifier: 'c3i@master', changelog: false, - retriever: modernSCM([$class: 'GitSCMSource', remote: 'https://pagure.io/c3i-library.git']) -def label = "jenkins-slave-${UUID.randomUUID().toString()}" -podTemplate( - cloud: "${params.JENKINS_AGENT_CLOUD_NAME}", - label: label, - serviceAccount: "${env.JENKINS_AGENT_SERVICE_ACCOUNT}", - yaml: """ - apiVersion: v1 - kind: Pod - metadata: - labels: - app: "jenkins-${env.JOB_BASE_NAME.take(50)}" - factory2-pipeline-kind: "mbs-repotracker-trigger" - factory2-pipeline-build-number: "${env.BUILD_NUMBER}" - spec: - containers: - - name: jnlp - image: ${params.JENKINS_AGENT_IMAGE} - imagePullPolicy: Always - tty: true - env: - - name: REGISTRY_CREDENTIALS - valueFrom: - secretKeyRef: - name: "${params.CONTAINER_REGISTRY_CREDENTIALS}" - key: ".dockerconfigjson" - resources: - requests: - memory: 256Mi - cpu: 200m - limits: - memory: 512Mi - cpu: 300m - """ -) { - node(label) { - stage('Allocate C3IaaS project') { - if (!(params.USE_C3IAAS == 'true' && - params.C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE && - params.C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAME)) { - echo "Not configured to use C3IaaS" - return - } - env.PIPELINE_NAMESPACE = readFile("/run/secrets/kubernetes.io/serviceaccount/namespace").trim() - env.C3IAAS_NAMESPACE = "c3i-mbs-tr-${params.TRACKED_TAG}-${env.BUILD_NUMBER}" - echo "Requesting new OpenShift project ${env.C3IAAS_NAMESPACE}..." - openshift.withCluster() { - openshift.withProject(params.C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE) { - c3i.buildAndWait(script: this, objs: "bc/${params.C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAME}", - '-e', "PROJECT_NAME=${env.C3IAAS_NAMESPACE}", - '-e', "ADMIN_GROUPS=system:serviceaccounts:${env.PIPELINE_NAMESPACE}", - '-e', "LIFETIME_IN_MINUTES=${params.C3IAAS_LIFETIME}" - ) - } - } - echo "Allocated project ${env.C3IAAS_NAMESPACE}" - } - stage('Trigger tests') { - def message = readJSON text: params.CI_MESSAGE - echo "Tag :${message.tag} is ${message.action} in ${message.repo}. New digest: ${message.digest}" - def frontendImage = "${message.repo}@${message.digest}" - // We have the digest of the current frontend image with this tag. - // Lookup the digest of the current backend image with the same tag. - if (env.REGISTRY_CREDENTIALS) { - dir ("${env.HOME}/.docker") { - writeFile file: 'config.json', text: env.REGISTRY_CREDENTIALS - } - } - def output = sh(script: "skopeo inspect docker://${params.MBS_BACKEND_REPO}:${message.tag}", returnStdout: true).trim() - def backendData = readJSON text: output - def backendImage = "${params.MBS_BACKEND_REPO}@${backendData.Digest}" - echo "Current mbs-backend image is: ${backendImage}" - echo "Triggering a job to test if ${frontendImage} and ${backendImage} meet all criteria of desired tag" - openshift.withCluster() { - openshift.withProject(params.TEST_JOB_NAMESPACE) { - def build = c3i.build(script: this, objs: "bc/${params.TEST_JOB_NAME}", - '-e', "MBS_BACKEND_IMAGE=${backendImage}", - '-e', "MBS_FRONTEND_IMAGE=${frontendImage}", - '-e', "TEST_IMAGES=${backendImage},${frontendImage}", - '-e', "IMAGE_IS_SCRATCH=false", - '-e', "TEST_NAMESPACE=${env.C3IAAS_NAMESPACE ?: ''}", - ) - c3i.waitForBuildStart(script: this, build: build) - buildInfo = build.object() - echo "Build ${buildInfo.metadata.annotations['openshift.io/jenkins-build-uri'] ?: buildInfo.metadata.name} started." +{% include "snippets/c3i-library.groovy" %} +pipeline { + {% include "snippets/default-agent.groovy" %} + options { + timestamps() + timeout(time: 120, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '10')) + } + triggers { + ciBuildTrigger( + noSquash: false, + providerList: [ + activeMQSubscriber( + name: params.MESSAGING_PROVIDER, + overrides: [topic: params.MESSAGING_TOPIC], + selector: "repo = '${params.TRACKED_CONTAINER_REPO}' AND action IN ('added', 'updated') AND tag = '${params.TRACKED_TAG}'", + ) + ] + ) + } + stages { + stage("Message Check and setup") { + steps { + script { + if (!params.CI_MESSAGE) { + error("This build is not started by a CI message. Only configurations were done.") + } + c3i.clone(repo: params.MBS_GIT_REPO, branch: params.MBS_GIT_REF) + def message = readJSON text: params.CI_MESSAGE + echo "Tag :${message.tag} is ${message.action} in ${message.repo}. New digest: ${message.digest}" + env.FRONTEND_IMAGE_REF = "${message.repo}@${message.digest}" + // We have the digest of the current frontend image with this tag. + // Lookup the digest of the current backend image with the same tag. + if (params.CONTAINER_REGISTRY_CREDENTIALS) { + dir ("${env.HOME}/.docker") { + openshift.withCluster() { + def dockerconf = openshift.selector('secret', params.CONTAINER_REGISTRY_CREDENTIALS).object().data['.dockerconfigjson'] + writeFile file: 'config.json', text: dockerconf, encoding: "Base64" + } + } + } + def output = sh(script: "skopeo inspect docker://${params.MBS_BACKEND_REPO}:${message.tag}", returnStdout: true).trim() + def backendData = readJSON text: output + env.BACKEND_IMAGE_REF = "${params.MBS_BACKEND_REPO}@${backendData.Digest}" + echo "Current mbs-backend image is: ${env.BACKEND_IMAGE_REF}" + echo "Triggering a job to test if ${env.FRONTEND_IMAGE_REF} and ${env.BACKEND_IMAGE_REF} meet all criteria of desired tag" + env.C3IAAS_PROJECT = params.C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE + env.IMAGE_IS_SCRATCH = false + env.PIPELINE_ID = "c3i-mbs-tag-${message.tag}-${message.digest[-9..-1]}" } } } + {% include "snippets/mbs-integration-test.groovy" %} } } +{% include "snippets/functions.groovy" %} diff --git a/openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.yaml b/openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.yaml index f9018ca8..6a373813 100644 --- a/openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.yaml +++ b/openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.yaml @@ -15,10 +15,12 @@ parameters: - name: MBS_GIT_REPO displayName: MBS Git repo URL description: Default MBS Git repo URL in which to run dev tests against - value: "https://pagure.io/fm-orchestrator.git" + required: true + value: https://pagure.io/fm-orchestrator.git - name: MBS_GIT_REF displayName: MBS Git repo ref description: Default MBS Git repo ref in which to run dev tests against + required: true value: master - name: TRACKED_CONTAINER_REPO displayName: Container repo to be tracked @@ -45,12 +47,10 @@ parameters: - name: MESSAGING_TOPIC displayName: Name of the topic that the trigger subscribes to value: "Consumer.rh-jenkins-ci-plugin.c3i-mbs-repotracker-trigger.VirtualTopic.eng.repotracker.container.tag.>" -- name: TEST_JOB_NAME - displayName: Name of integration test job to trigger +- name: ENVIRONMENT + displayName: environment name (dev/stage/prod) required: true -- name: TEST_JOB_NAMESPACE - displayName: Namespace in which to trigger the integration test job - required: false + value: dev - name: USE_C3IAAS displayName: >- Use C3I-as-a-Service to dynamically allocate a temporary OpenShift project for building @@ -69,6 +69,11 @@ parameters: displayName: The lifetime of the OpenShift project allocated by C3I-as-a-Service. required: true value: "120" +- name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE + displayName: The namespace where the Pipeline-as-a-Service project request BuildConfig has been defined + required: false + value: c3i +{% include "snippets/c3i-library-parameters.yaml" %} objects: - kind: ServiceAccount apiVersion: v1 @@ -96,10 +101,6 @@ objects: spec: runPolicy: "Parallel" completionDeadlineSeconds: 1800 - source: - git: - uri: "${MBS_GIT_REPO}" - ref: "${MBS_GIT_REF}" strategy: type: JenkinsPipeline jenkinsPipelineStrategy: @@ -110,6 +111,10 @@ objects: value: "${JENKINS_AGENT_IMAGE}" - name: JENKINS_AGENT_SERVICE_ACCOUNT value: "${NAME}-jenkins-slave" + - name: MBS_GIT_REPO + value: "${MBS_GIT_REPO}" + - name: MBS_GIT_REF + value: "${MBS_GIT_REF}" - name: TRACKED_CONTAINER_REPO value: "${TRACKED_CONTAINER_REPO}" - name: TRACKED_TAG @@ -118,18 +123,18 @@ objects: value: "${MBS_BACKEND_REPO}" - name: CONTAINER_REGISTRY_CREDENTIALS value: "${CONTAINER_REGISTRY_CREDENTIALS}" - - name: TEST_JOB_NAME - value: "${TEST_JOB_NAME}" - - name: TEST_JOB_NAMESPACE - value: "${TEST_JOB_NAMESPACE}" - name: USE_C3IAAS value: "${USE_C3IAAS}" + - name: ENVIRONMENT + value: "${ENVIRONMENT}" - name: C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE value: "${C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE}" - name: C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAME value: "${C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAME}" - name: C3IAAS_LIFETIME value: "${C3IAAS_LIFETIME}" + - name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE + value: "${PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE}" - name: MESSAGING_PROVIDER value: "${MESSAGING_PROVIDER}" - name: MESSAGING_TOPIC @@ -139,4 +144,5 @@ objects: value: - name: MESSAGE_HEADERS value: - jenkinsfilePath: openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.Jenkinsfile + jenkinsfile: | + {% filter indent(width=10) %}{% include "mbs-repotracker-trigger.Jenkinsfile" %}{% endfilter %} diff --git a/openshift/integration/koji/pipelines/templates/snippets/c3i-library-parameters.yaml b/openshift/integration/koji/pipelines/templates/snippets/c3i-library-parameters.yaml new file mode 100644 index 00000000..5f96d609 --- /dev/null +++ b/openshift/integration/koji/pipelines/templates/snippets/c3i-library-parameters.yaml @@ -0,0 +1,8 @@ +- name: C3I_LIB_URL + displayName: C3I library git url + required: true + value: "https://pagure.io/c3i-library.git" +- name: C3I_LIB_BRANCH + displayName: C3I library branch + required: true + value: "master" diff --git a/openshift/integration/koji/pipelines/templates/snippets/c3i-library.groovy b/openshift/integration/koji/pipelines/templates/snippets/c3i-library.groovy new file mode 100644 index 00000000..e752c819 --- /dev/null +++ b/openshift/integration/koji/pipelines/templates/snippets/c3i-library.groovy @@ -0,0 +1,2 @@ +library identifier: "c3i@${C3I_LIB_BRANCH}", changelog: false, + retriever: modernSCM([$class: 'GitSCMSource', remote: "${C3I_LIB_URL}"]) diff --git a/openshift/integration/koji/pipelines/templates/snippets/default-agent.groovy b/openshift/integration/koji/pipelines/templates/snippets/default-agent.groovy new file mode 100644 index 00000000..919259dc --- /dev/null +++ b/openshift/integration/koji/pipelines/templates/snippets/default-agent.groovy @@ -0,0 +1,29 @@ +agent { + kubernetes { + cloud "${params.JENKINS_AGENT_CLOUD_NAME}" + label "jenkins-slave-${UUID.randomUUID().toString()}" + serviceAccount "${params.JENKINS_AGENT_SERVICE_ACCOUNT}" + defaultContainer 'jnlp' + yaml """ + apiVersion: v1 + kind: Pod + metadata: + labels: + app: "jenkins-${env.JOB_BASE_NAME.take(50).endsWith('-') ? env.JOB_BASE_NAME.take(49): env.JOB_BASE_NAME.take(50)}" + factory2-pipeline-build-number: "${env.BUILD_NUMBER}" + spec: + containers: + - name: jnlp + image: "${params.JENKINS_AGENT_IMAGE}" + imagePullPolicy: Always + tty: true + resources: + requests: + memory: 512Mi + cpu: 300m + limits: + memory: 768Mi + cpu: 500m + """ + } +} diff --git a/openshift/integration/koji/pipelines/templates/snippets/functions.groovy b/openshift/integration/koji/pipelines/templates/snippets/functions.groovy new file mode 100644 index 00000000..1f12fc92 --- /dev/null +++ b/openshift/integration/koji/pipelines/templates/snippets/functions.groovy @@ -0,0 +1,64 @@ +def sendToResultsDB(imageRef, status) { + if (!params.MESSAGING_PROVIDER) { + echo "Message bus is not set. Skipping send of:\nimageRef: ${imageRef}\nstatus: ${status}" + return + } + def (repourl, digest) = imageRef.tokenize('@') + def (registry, reponame) = repourl.split('/', 2) + def image = reponame.split('/').last() + def sendResult = sendCIMessage \ + providerName: params.MESSAGING_PROVIDER, \ + overrides: [topic: 'VirtualTopic.eng.ci.container-image.test.complete'], \ + messageType: 'Custom', \ + messageProperties: '', \ + messageContent: """ + { + "ci": { + "name": "C3I Jenkins", + "team": "DevOps", + "url": "${env.JENKINS_URL}", + "docs": "https://pagure.io/fm-orchestrator/blob/master/f/openshift/integration/koji", + "irc": "#pnt-devops-dev", + "email": "pnt-factory2-devel@redhat.com", + "environment": "${params.ENVIRONMENT}" + }, + "run": { + "url": "${env.BUILD_URL}", + "log": "${env.BUILD_URL}/console", + "debug": "", + "rebuild": "${env.BUILD_URL}/rebuild/parametrized" + }, + "artifact": { + "type": "container-image", + "repository": "${reponame}", + "digest": "${digest}", + "nvr": "${imageRef}", + "issuer": "c3i-jenkins", + "scratch": ${env.IMAGE_IS_SCRATCH}, + "id": "${image}@${digest}" + }, + "system": + [{ + "os": "${params.JENKINS_AGENT_IMAGE}", + "provider": "openshift", + "architecture": "x86_64" + }], + "type": "integration", + "category": "${params.ENVIRONMENT}", + "status": "${status}", + "xunit": "", + "generated_at": "${new Date().format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone('UTC'))}", + "namespace": "c3i", + "version": "0.1.0" + } + """ + if (sendResult.getMessageId()) { + // echo sent message id and content + echo 'Successfully sent the test result to ResultsDB.' + echo "Message ID: ${sendResult.getMessageId()}" + echo "Message content: ${sendResult.getMessageContent()}" + } else { + echo 'Failed to sent the test result to ResultsDB.' + } +} + diff --git a/openshift/integration/koji/pipelines/templates/snippets/get_paas_domain.groovy b/openshift/integration/koji/pipelines/templates/snippets/get_paas_domain.groovy new file mode 100644 index 00000000..9733732c --- /dev/null +++ b/openshift/integration/koji/pipelines/templates/snippets/get_paas_domain.groovy @@ -0,0 +1,14 @@ +if (!env.TRIGGER_NAMESPACE) { + env.TRIGGER_NAMESPACE = readFile("/run/secrets/kubernetes.io/serviceaccount/namespace").trim() +} +if(!env.PAAS_DOMAIN) { + openshift.withCluster() { + openshift.withProject(env.TRIGGER_NAMESPACE) { + def testroute = openshift.create('route', 'edge', "test-${env.BUILD_NUMBER}", '--service=test', '--port=8080') + def testhost = testroute.object().spec.host + env.PAAS_DOMAIN = testhost.minus("test-${env.BUILD_NUMBER}-${env.TRIGGER_NAMESPACE}.") + testroute.delete() + } + } + echo "Routes end with ${env.PAAS_DOMAIN}" +} diff --git a/openshift/integration/koji/pipelines/templates/snippets/mbs-integration-test.groovy b/openshift/integration/koji/pipelines/templates/snippets/mbs-integration-test.groovy new file mode 100644 index 00000000..d71bca59 --- /dev/null +++ b/openshift/integration/koji/pipelines/templates/snippets/mbs-integration-test.groovy @@ -0,0 +1,103 @@ +stage('Run integration tests') { + stages { + stage('Deploy test environment') { + steps { + script { + {% include "snippets/get_paas_domain.groovy" %} + if (!env.PIPELINE_ID) { + env.PIPELINE_ID = "c3i-mbs-${UUID.randomUUID().toString().take(8)}" + } + openshift.withCluster() { + openshift.withProject(params.PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE) { + c3i.buildAndWait(script: this, objs: "bc/pipeline-as-a-service", + '-e', "DEFAULT_IMAGE_TAG=${env.ENVIRONMENT}", + '-e', "PIPELINE_ID=${env.PIPELINE_ID}", + '-e', "WAIVERDB_IMAGE=", + '-e', "C3IAAS_PROJECT=${env.C3IAAS_PROJECT ?: ''}", + '-e', "RESULTSDB_IMAGE=", + '-e', "RESULTSDB_UPDATER_IMAGE=", + '-e', "GREENWAVE_IMAGE=", + '-e', "DATAGREPPER_IMAGE=", + '-e', "DATANOMMER_IMAGE=", + '-e', "MBS_BACKEND_IMAGE=${env.BACKEND_IMAGE_REF}", + '-e', "MBS_FRONTEND_IMAGE=${env.FRONTEND_IMAGE_REF}", + '-e', "PAAS_DOMAIN=${env.PAAS_DOMAIN}" + ) + } + } + } + } + } + stage('Run tests') { + steps { + script { + def testcases + if (params.TESTCASES) { + if (params.TESTCASES == 'skip') { + testcases = [] + echo 'Skipping integration tests' + } else { + testcases = params.TESTCASES.split() + echo "Using specified list of testcases: ${testcases}" + } + } else { + testcases = findFiles(glob: 'openshift/integration/koji/pipelines/tests/*.groovy').collect { + it.name.minus('.groovy') + } + echo "Using all available testcases: ${testcases}" + } + testcases.each { testcase -> + env.CURRENT_TESTCASE = testcase + echo "Running testcase ${testcase}..." + def test = load "openshift/integration/koji/pipelines/tests/${testcase}.groovy" + test.runTests() + } + } + } + post { + success { + echo "All tests successful" + script { + [env.BACKEND_IMAGE_REF, env.FRONTEND_IMAGE_REF].each { + sendToResultsDB(it, 'passed') + } + } + } + failure { + echo "Testcase ${env.CURRENT_TESTCASE} FAILED" + } + } + } + } + post { + failure { + script { + [env.BACKEND_IMAGE_REF, env.FRONTEND_IMAGE_REF].each { + sendToResultsDB(it, 'failed') + } + openshift.withCluster() { + openshift.withProject(env.PIPELINE_ID) { + echo 'Getting logs from all deployments...' + openshift.selector('pods', ['c3i.redhat.com/pipeline': env.PIPELINE_ID]).logs('--tail 100') + } + } + } + } + cleanup { + script { + if (params.CLEANUP == 'true' && params.USE_C3IAAS != 'true') { + openshift.withCluster() { + openshift.withProject(env.PIPELINE_ID) { + /* Tear down everything we just created */ + echo 'Tearing down test resources...' + openshift.selector('all,pvc,configmap,secret', + ['c3i.redhat.com/pipeline': env.PIPELINE_ID]).delete('--ignore-not-found=true') + } + } + } else { + echo 'Skipping cleanup' + } + } + } + } +}