modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-05 11:48:33 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge #1545 Use jinja templates to provide 'full-jobs'

Browse Source
This commit is contained in:
Matt Prahl
2019-12-12 15:05:14 +00:00
parent f61f7afbdd a97f6ce9ff
commit 7c91d7e2e1
40 changed files with 596 additions and 1123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
openshift/integration/koji/containers/jenkins-slave/Dockerfile
View File

@@ -32,7 +32,7 @@ USER root
RUN ${DNF_CMD} install -y \
java-1.8.0-openjdk nss_wrapper gettext git \
tar gzip skopeo wget make bind-utils \
tar gzip skopeo wget make bind-utils python3-jinja2-cli \
origin-clients \
# Jenkins pipeline 'sh' steps seem to require ps
procps-ng \

2
openshift/integration/koji/pipelines/Makefile
View File

@@ -23,7 +23,7 @@ install:
@for job in $(JOBS); do \
echo "[PIPELINE] Updating pipeline job \"$${job}\"..." ; \
template_file=$$(cat ./$(JOBS_DIR)/$${job}.tmpl); \
$(OC_CMD) process --local -f ./$(TEMPLATES_DIR)/$${template_file} \
jinja2 ./$(TEMPLATES_DIR)/$${template_file} | $(OC_CMD) process --local -f - \
--param-file ./$(JOBS_DIR)/$${job}.env | $(OC_CMD) apply -f -; \
echo "[PIPELINE] Pipeline job \"$${job}\" updated" ; \
done

4
openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-prod.env
View File

@@ -4,4 +4,6 @@ SUBJECT_IDENTIFIER_REGEX=^factory2/mbs-backend@sha256:
SOURCE_CONTAINER_REPO=quay.io/factory2/mbs-backend
TARGET_TAG=prod
MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-backend-greenwave-promote-to-prod.VirtualTopic.eng.greenwave.decision.update
IMAGE_PROMOTION_JOB=mbs-backend-promoting-to-prod
PROMOTING_DESTINATIONS=quay.io/factory2/mbs-backend
TAG_INTO_IMAGESTREAM=true
DEST_IMAGESTREAM_NAME=mbs-backend

4
openshift/integration/koji/pipelines/jobs/mbs-backend-greenwave-promote-to-stage.env
View File

@@ -4,4 +4,6 @@ SUBJECT_IDENTIFIER_REGEX=^factory2/mbs-backend@sha256:
SOURCE_CONTAINER_REPO=quay.io/factory2/mbs-backend
TARGET_TAG=stage
MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-backend-greenwave-promote-to-stage.VirtualTopic.eng.greenwave.decision.update
IMAGE_PROMOTION_JOB=mbs-backend-promoting-to-stage
PROMOTING_DESTINATIONS=quay.io/factory2/mbs-backend
TAG_INTO_IMAGESTREAM=true
DEST_IMAGESTREAM_NAME=mbs-backend

5
openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-prod.env
View File

@@ -1,5 +0,0 @@
NAME=mbs-backend-promoting-to-prod
PROMOTING_DESTINATIONS=quay.io/factory2/mbs-backend
DEST_TAG=prod
TAG_INTO_IMAGESTREAM=true
DEST_IMAGESTREAM_NAME=mbs-backend

1
openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-prod.tmpl
View File

@@ -1 +0,0 @@
mbs-image-promotion-template.yaml

5
openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-stage.env
View File

@@ -1,5 +0,0 @@
NAME=mbs-backend-promoting-to-stage
PROMOTING_DESTINATIONS=quay.io/factory2/mbs-backend
DEST_TAG=stage
TAG_INTO_IMAGESTREAM=true
DEST_IMAGESTREAM_NAME=mbs-backend

1
openshift/integration/koji/pipelines/jobs/mbs-backend-promoting-to-stage.tmpl
View File

@@ -1 +0,0 @@
mbs-image-promotion-template.yaml

1
openshift/integration/koji/pipelines/jobs/mbs-dev-integration-test.env
View File

@@ -1 +0,0 @@
NAME=mbs-dev-integration-test

1
openshift/integration/koji/pipelines/jobs/mbs-dev-integration-test.tmpl
View File

@@ -1 +0,0 @@
mbs-integration-test-template.yaml

4
openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-prod.env
View File

@@ -4,4 +4,6 @@ SUBJECT_IDENTIFIER_REGEX=^factory2/mbs-frontend@sha256:
SOURCE_CONTAINER_REPO=quay.io/factory2/mbs-frontend
TARGET_TAG=prod
MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-frontend-greenwave-promote-to-prod.VirtualTopic.eng.greenwave.decision.update
IMAGE_PROMOTION_JOB=mbs-frontend-promoting-to-prod
PROMOTING_DESTINATIONS=quay.io/factory2/mbs-frontend
TAG_INTO_IMAGESTREAM=true
DEST_IMAGESTREAM_NAME=mbs-frontend

4
openshift/integration/koji/pipelines/jobs/mbs-frontend-greenwave-promote-to-stage.env
View File

@@ -4,4 +4,6 @@ SUBJECT_IDENTIFIER_REGEX=^factory2/mbs-frontend@sha256:
SOURCE_CONTAINER_REPO=quay.io/factory2/mbs-frontend
TARGET_TAG=stage
MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-frontend-greenwave-promote-to-stage.VirtualTopic.eng.greenwave.decision.update
IMAGE_PROMOTION_JOB=mbs-frontend-promoting-to-stage
PROMOTING_DESTINATIONS=quay.io/factory2/mbs-frontend
TAG_INTO_IMAGESTREAM=true
DEST_IMAGESTREAM_NAME=mbs-frontend

5
openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-prod.env
View File

@@ -1,5 +0,0 @@
NAME=mbs-frontend-promoting-to-prod
PROMOTING_DESTINATIONS=quay.io/factory2/mbs-frontend
DEST_TAG=prod
TAG_INTO_IMAGESTREAM=true
DEST_IMAGESTREAM_NAME=mbs-frontend

1
openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-prod.tmpl
View File

@@ -1 +0,0 @@
mbs-image-promotion-template.yaml

5
openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-stage.env
View File

@@ -1,5 +0,0 @@
NAME=mbs-frontend-promoting-to-stage
PROMOTING_DESTINATIONS=quay.io/factory2/mbs-frontend
DEST_TAG=stage
TAG_INTO_IMAGESTREAM=true
DEST_IMAGESTREAM_NAME=mbs-frontend

1
openshift/integration/koji/pipelines/jobs/mbs-frontend-promoting-to-stage.tmpl
View File

@@ -1 +0,0 @@
mbs-image-promotion-template.yaml

2
openshift/integration/koji/pipelines/jobs/mbs-prod-integration-test.env
View File

@@ -1,2 +0,0 @@
NAME=mbs-prod-integration-test
ENVIRONMENT=prod

1
openshift/integration/koji/pipelines/jobs/mbs-prod-integration-test.tmpl
View File

@@ -1 +0,0 @@
mbs-integration-test-template.yaml

2
openshift/integration/koji/pipelines/jobs/mbs-stage-integration-test.env
View File

@@ -1,2 +0,0 @@
NAME=mbs-stage-integration-test
ENVIRONMENT=stage

1
openshift/integration/koji/pipelines/jobs/mbs-stage-integration-test.tmpl
View File

@@ -1 +0,0 @@
mbs-integration-test-template.yaml

2
openshift/integration/koji/pipelines/jobs/mbs-trigger-on-latest-tag.env
View File

@@ -1,4 +1,4 @@
NAME=mbs-trigger-on-latest-tag
MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-trigger-on-latest-tag.VirtualTopic.eng.repotracker.container.tag.>
TEST_JOB_NAME=mbs-stage-integration-test
ENVIRONMENT=stage
TRACKED_TAG=latest

2
openshift/integration/koji/pipelines/jobs/mbs-trigger-on-stage-tag.env
View File

@@ -1,4 +1,4 @@
NAME=mbs-trigger-on-stage-tag
MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-mbs-trigger-on-stage-tag.VirtualTopic.eng.repotracker.container.tag.>
TEST_JOB_NAME=mbs-prod-integration-test
TRACKED_TAG=stage
ENVIRONMENT=prod

31
openshift/integration/koji/pipelines/templates/mbs-build-template.yaml
View File

@@ -90,13 +90,6 @@ parameters:
- name: MBS_FRONTEND_IMAGESTREAM_NAMESPACE
displayName: Namespace of ImageStream for MBS frontend images
required: false
- name: MBS_INTEGRATION_TEST_BUILD_CONFIG_NAME
displayName: Name of BuildConfig for running integration tests
required: true
value: mbs-dev-integration-test
- name: MBS_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE
displayName: Namespace of BuildConfig for running integration tests
required: false
- name: FORCE_PUBLISH_IMAGE
displayName: Whether to push the resulting image regardless of the Git branch
value: "false"
@@ -153,10 +146,19 @@ parameters:
displayName: The lifetime of the OpenShift project allocated by C3I-as-a-Service.
required: true
value: "120"
- name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE
displayName: The namespace where the Pipeline-as-a-Service project request BuildConfig has been defined
required: false
value: c3i
- name: CLEANUP
displayName: Cleanup objects after the pipeline is complete
required: true
value: "true"
- name: ENVIRONMENT
displayName: environment name (dev/stage/prod)
required: true
value: dev
{% include "snippets/c3i-library-parameters.yaml" %}
labels:
template: mbs-build
objects:
@@ -223,10 +225,6 @@ objects:
spec:
runPolicy: "Parallel"
completionDeadlineSeconds: 1800
source:
git:
uri: "${MBS_GIT_REPO}"
ref: "${MBS_GIT_REF}"
strategy:
type: JenkinsPipeline
jenkinsPipelineStrategy:
@@ -263,10 +261,6 @@ objects:
value: "${MBS_FRONTEND_IMAGESTREAM_NAMESPACE}"
- name: MBS_MAIN_BRANCH
value: "${MBS_MAIN_BRANCH}"
- name: MBS_INTEGRATION_TEST_BUILD_CONFIG_NAME
value: "${MBS_INTEGRATION_TEST_BUILD_CONFIG_NAME}"
- name: MBS_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE
value: "${MBS_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE}"
- name: PAGURE_REPO_NAME
value: "${PAGURE_REPO_NAME}"
- name: PAGURE_REPO_IS_FORK
@@ -285,6 +279,8 @@ objects:
value: "${EXTRA_RPMS}"
- name: TESTCASES
value: "${TESTCASES}"
- name: ENVIRONMENT
value: "${ENVIRONMENT}"
- name: USE_C3IAAS
value: "${USE_C3IAAS}"
- name: C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE
@@ -293,6 +289,9 @@ objects:
value: "${C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAME}"
- name: C3IAAS_LIFETIME
value: "${C3IAAS_LIFETIME}"
- name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE
value: "${PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE}"
- name: CLEANUP
value: "${CLEANUP}"
jenkinsfilePath: openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile
jenkinsfile: |
{% filter indent(width=10) %}{% include "mbs-build.Jenkinsfile" %}{% endfilter %}

90
openshift/integration/koji/pipelines/templates/mbs-build.Jenkinsfile
View File

@@ -1,43 +1,7 @@
library identifier: 'c3i@master', changelog: false,
retriever: modernSCM([$class: 'GitSCMSource', remote: 'https://pagure.io/c3i-library.git'])
{% include "snippets/c3i-library.groovy" %}
import static org.apache.commons.lang.StringEscapeUtils.escapeHtml;
pipeline {
agent {
kubernetes {
cloud params.JENKINS_AGENT_CLOUD_NAME
label "jenkins-slave-${UUID.randomUUID().toString()}"
serviceAccount params.JENKINS_AGENT_SERVICE_ACCOUNT
defaultContainer 'jnlp'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
app: "jenkins-${env.JOB_BASE_NAME.take(50)}"
factory2-pipeline-kind: "mbs-build-pipeline"
factory2-pipeline-build-number: "${env.BUILD_NUMBER}"
spec:
containers:
- name: jnlp
image: "${params.JENKINS_AGENT_IMAGE}"
imagePullPolicy: Always
tty: true
env:
- name: REGISTRY_CREDENTIALS
valueFrom:
secretKeyRef:
name: "${params.CONTAINER_REGISTRY_CREDENTIALS}"
key: ".dockerconfigjson"
resources:
requests:
memory: 512Mi
cpu: 300m
limits:
memory: 768Mi
cpu: 500m
"""
}
}
{% include "snippets/default-agent.groovy" %}
options {
timestamps()
timeout(time: 120, unit: 'MINUTES')
@@ -64,6 +28,7 @@ pipeline {
env.GIT_COMMIT = env.MBS_GIT_COMMIT
echo "Build ${params.MBS_GIT_REF}, commit=${env.MBS_GIT_COMMIT}"
env.IMAGE_IS_SCRATCH = (params.MBS_GIT_REF != params.MBS_MAIN_BRANCH)
// Is the current branch a pull-request? If no, env.PR_NO will be empty.
env.PR_NO = getPrNo(params.MBS_GIT_REF)
@@ -102,6 +67,7 @@ pipeline {
-e '/sitepackages/a setenv = PYTHONPATH={toxinidir}' \
tox.ini
"""
{% include "snippets/get_paas_domain.groovy" %}
}
}
}
@@ -304,33 +270,7 @@ pipeline {
}
}
}
stage('Run integration tests') {
steps {
script {
openshift.withCluster() {
openshift.withProject(params.MBS_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE) {
def build = c3i.buildAndWait(script: this, objs: "bc/${params.MBS_INTEGRATION_TEST_BUILD_CONFIG_NAME}",
'-e', "MBS_GIT_REPO=${params.MBS_GIT_REPO}",
'-e', "MBS_GIT_REF=${env.PR_NO ? params.MBS_GIT_REF : env.MBS_GIT_COMMIT}",
'-e', "MBS_BACKEND_IMAGE=${env.BACKEND_IMAGE_REF}",
'-e', "MBS_FRONTEND_IMAGE=${env.FRONTEND_IMAGE_REF}",
'-e', "TEST_IMAGES=${env.BACKEND_IMAGE_REF},${env.FRONTEND_IMAGE_REF}",
'-e', "IMAGE_IS_SCRATCH=${params.MBS_GIT_REF != params.MBS_MAIN_BRANCH}",
'-e', "TEST_NAMESPACE=${env.PIPELINE_ID}",
'-e', "TESTCASES='${params.TESTCASES}'",
'-e', "CLEANUP=${params.CLEANUP}"
)
echo 'Integration tests PASSED'
}
}
}
}
post {
failure {
echo 'Integration tests FAILED'
}
}
}
{% include "snippets/mbs-integration-test.groovy" %}
stage('Push images') {
when {
expression {
@@ -340,10 +280,13 @@ pipeline {
}
steps {
script {
if (env.REGISTRY_CREDENTIALS) {
dir ("${env.HOME}/.docker") {
writeFile file: 'config.json', text: env.REGISTRY_CREDENTIALS
}
if (params.CONTAINER_REGISTRY_CREDENTIALS) {
dir ("${env.HOME}/.docker") {
openshift.withCluster() {
def dockerconf = openshift.selector('secret', params.CONTAINER_REGISTRY_CREDENTIALS).object().data['.dockerconfigjson']
writeFile file: 'config.json', text: dockerconf, encoding: "Base64"
}
}
}
def registryToken = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/token')
def copyDown = { name, src ->
@@ -412,13 +355,13 @@ pipeline {
steps {
script {
openshift.withCluster() {
openshift.withProject(params.MBS_BACKEND_IMAGESTREAM_NAMESPACE) {
openshift.withProject(params.MBS_BACKEND_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_ID) {
def sourceRef = "${params.MBS_BACKEND_IMAGESTREAM_NAME}@${env.BACKEND_IMAGE_DIGEST}"
def destRef = "${params.MBS_BACKEND_IMAGESTREAM_NAME}:${params.MBS_DEV_IMAGE_TAG}"
echo "Tagging ${sourceRef} as ${destRef}..."
openshift.tag(sourceRef, destRef)
}
openshift.withProject(params.MBS_FRONTEND_IMAGESTREAM_NAMESPACE) {
openshift.withProject(params.MBS_FRONTEND_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_ID) {
def sourceRef = "${params.MBS_FRONTEND_IMAGESTREAM_NAME}@${env.FRONTEND_IMAGE_DIGEST}"
def destRef = "${params.MBS_FRONTEND_IMAGESTREAM_NAME}:${params.MBS_DEV_IMAGE_TAG}"
echo "Tagging ${sourceRef} as ${destRef}..."
@@ -442,13 +385,13 @@ pipeline {
openshift.withProject(env.PIPELINE_ID) {
if (env.BACKEND_IMAGE_TAG) {
echo "Removing tag ${env.BACKEND_IMAGE_TAG} from the ${params.MBS_BACKEND_IMAGESTREAM_NAME} ImageStream..."
openshift.withProject(params.MBS_BACKEND_IMAGESTREAM_NAMESPACE) {
openshift.withProject(params.MBS_BACKEND_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_ID) {
openshift.tag("${params.MBS_BACKEND_IMAGESTREAM_NAME}:${env.BACKEND_IMAGE_TAG}", "-d")
}
}
if (env.FRONTEND_IMAGE_TAG) {
echo "Removing tag ${env.FRONTEND_IMAGE_TAG} from the ${params.MBS_FRONTEND_IMAGESTREAM_NAME} ImageStream..."
openshift.withProject(params.MBS_FRONTEND_IMAGESTREAM_NAMESPACE) {
openshift.withProject(params.BS_FRONTEND_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_ID) {
openshift.tag("${params.MBS_FRONTEND_IMAGESTREAM_NAME}:${env.FRONTEND_IMAGE_TAG}", "-d")
}
}
@@ -539,3 +482,4 @@ def sendBuildStatusEmail(String status) {
}
emailext to: recipient, subject: subject, body: body
}
{% include "snippets/functions.groovy" %}

193
openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.Jenkinsfile
View File

@@ -1,79 +1,122 @@
// Use scripted syntax because CIBuildTrigger currently doesn't support the declarative syntax
properties([
buildDiscarder(logRotator(numToKeepStr: '10')),
pipelineTriggers([
// example: https://github.com/jenkinsci/jms-messaging-plugin/blob/9b9387c3a52f037ba0d019c2ebcf2a2796fc6397/src/test/java/com/redhat/jenkins/plugins/ci/integration/AmqMessagingPluginIntegrationTest.java
[$class: 'CIBuildTrigger',
providerData: [$class: 'ActiveMQSubscriberProviderData',
name: params.MESSAGING_PROVIDER,
overrides: [topic: params.MESSAGING_TOPIC],
checks: [
[field: '$.msg.subject_type', expectedValue: 'container-image'],
[field: '$.msg.subject_identifier', expectedValue: params.SUBJECT_IDENTIFIER_REGEX],
[field: '$.msg.decision_context', expectedValue: params.DECISION_CONTEXT_REGEX],
[field: '$.msg.policies_satisfied', expectedValue: 'true'],
],
],
],
]),
])
if (!params.CI_MESSAGE) {
echo 'This build is not started by a CI message. Only configurations were done.'
return
}
def label = "jenkins-slave-${UUID.randomUUID().toString()}"
podTemplate(
cloud: "${params.JENKINS_AGENT_CLOUD_NAME}",
label: label,
serviceAccount: "${params.JENKINS_AGENT_SERVICE_ACCOUNT}",
defaultContainer: 'jnlp',
yaml: """
apiVersion: v1
kind: Pod
metadata:
labels:
app: "jenkins-${env.JOB_BASE_NAME.take(50)}"
factory2-pipeline-kind: "mbs-greenwave-trigger"
factory2-pipeline-build-number: "${env.BUILD_NUMBER}"
spec:
containers:
- name: jnlp
image: ${params.JENKINS_AGENT_IMAGE}
imagePullPolicy: Always
tty: true
resources:
requests:
memory: 256Mi
cpu: 200m
limits:
memory: 512Mi
cpu: 300m
"""
) {
node(label) {
stage('Trigger promotion') {
def message = readJSON text: params.CI_MESSAGE
// Extract the digest of the image to be promoted.
// e.g. factory2/waiverdb@sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b
// -> sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b
def digest = (message.msg.subject_identifier =~ /@(sha256:\w+)$/)[0][1]
// Generate the pull spec of the image
// e.g. quay.io/factory2/waiverdb@sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b
def image = "${params.SOURCE_CONTAINER_REPO}@${digest}"
echo "Starting a new build to promote image ${image} to :${params.TARGET_TAG}..."
openshift.withCluster() {
def bcSelector = openshift.selector('bc', params.IMAGE_PROMOTION_JOB)
def buildSelector = bcSelector.startBuild(
'-e', "IMAGE=${image}",
'-e', "DEST_TAG=${params.TARGET_TAG}",
)
bcSelector.watch {
return !(it.object().status.phase in ["New", "Pending"])
{% include "snippets/c3i-library.groovy" %}
pipeline {
{% include "snippets/default-agent.groovy" %}
options {
timestamps()
timeout(time: 30, unit: 'MINUTES')
buildDiscarder(logRotator(numToKeepStr: '10'))
}
environment {
PIPELINE_NAMESPACE = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/namespace').trim()
SERVICE_ACCOUNT_TOKEN = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/token').trim()
}
triggers {
ciBuildTrigger(
noSquash: false,
providerList: [
activeMQSubscriber(
name: params.MESSAGING_PROVIDER,
overrides: [topic: params.MESSAGING_TOPIC],
checks: [
[field: '$.msg.subject_type', expectedValue: 'container-image'],
[field: '$.msg.subject_identifier', expectedValue: params.SUBJECT_IDENTIFIER_REGEX],
[field: '$.msg.decision_context', expectedValue: params.DECISION_CONTEXT_REGEX],
[field: '$.msg.policies_satisfied', expectedValue: 'true'],
]
)
]
)
}
stages {
stage("Message Check and setup") {
steps {
script {
if (!params.CI_MESSAGE) {
error("This build is not started by a CI message. Only configurations were done.")
}
buildInfo = buildSelector.object()
echo "Build ${buildInfo.metadata.annotations['openshift.io/jenkins-build-uri'] ?: buildInfo.metadata.name} started."
def message = readJSON text: params.CI_MESSAGE
// Extract the digest of the image to be promoted.
// e.g. factory2/waiverdb@sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b
// -> sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b
def digest = (message.msg.subject_identifier =~ /@(sha256:\w+)$/)[0][1]
// Generate the pull spec of the image
// e.g. quay.io/mkovarik/waiverdb@sha256:35201c572fc8a137862b7a256476add8d7465fa5043d53d117f4132402f8ef6b
env.IMAGE = "${params.SOURCE_CONTAINER_REPO}@${digest}"
echo "Starting promotion of image ${env.IMAGE} to :${params.TARGET_TAG}..."
// Setting up registry credentials
dir ("${env.HOME}/.docker") {
// for the OpenShift internal registry
def dockerConfig = readJSON text: '{ "auths": {} }'
dockerConfig.auths['docker-registry.default.svc:5000'] = [
'email': '',
'auth': sh(returnStdout: true, script: 'set +x; echo -n "serviceaccount:$SERVICE_ACCOUNT_TOKEN" | base64 -').trim()
]
// merging user specified credentials
if (params.CONTAINER_REGISTRY_CREDENTIALS) {
openshift.withCluster() {
def dockerconf = openshift.selector('secret', params.CONTAINER_REGISTRY_CREDENTIALS).object().data['.dockerconfigjson']
def dockerString = new String(dockerconf.decodeBase64())
toBeMerged = readJSON text: dockerString
dockerConfig.auths.putAll(toBeMerged.auths)
}
}
// writing to ~/.docker/config.json
writeJSON file: 'config.json', json: dockerConfig
}
}
}
}
stage('Pull image') {
steps {
echo "Pulling container image ${env.IMAGE}..."
withEnv(["SOURCE_IMAGE_REF=${env.IMAGE}"]) {
sh '''
set -e +x # hide the token from Jenkins console
mkdir -p _image
skopeo copy docker://"$SOURCE_IMAGE_REF" dir:_image
'''
}
}
}
stage('Promote image') {
steps {
script {
def destinations = params.PROMOTING_DESTINATIONS ? params.PROMOTING_DESTINATIONS.split(',') : []
openshift.withCluster() {
def pushTasks = destinations.collectEntries {
["Pushing ${it}" : {
def dest = "${it}:${params.TARGET_TAG}"
// Only docker and atomic registries are allowed
if (!dest.startsWith('atomic:') && !dest.startsWith('docker://')) {
dest = "docker://${dest}"
}
echo "Pushing container image to ${dest}..."
withEnv(["DEST_IMAGE_REF=${dest}"]) {
retry(5) {
sh 'skopeo copy dir:_image "$DEST_IMAGE_REF"'
}
}
}]
}
parallel pushTasks
}
}
}
}
stage('Tag ImageStream') {
when {
expression {
return params.DEST_IMAGESTREAM_NAME && params.TAG_INTO_IMAGESTREAM == "true"
}
}
steps {
script {
def destRef = "${params.DEST_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_NAMESPACE}/${params.DEST_IMAGESTREAM_NAME}:${params.TARGET_TAG}"
openshift.withCluster() {
echo "Tagging ${env.IMAGE} into ${destRef}..."
openshift.tag('--source=docker', env.IMAGE, destRef)
}
}
}
}
}

35
openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.yaml
View File

@@ -32,9 +32,18 @@ parameters:
- name: TARGET_TAG
displayName: Tag name to promote the image to
required: true
- name: IMAGE_PROMOTION_JOB
displayName: Downstream image promotion job to trigger
- name: TAG_INTO_IMAGESTREAM
displayName: Whether to tag the image into an ImageStream
value: "false"
required: true
- name: DEST_IMAGESTREAM_NAME
displayName: Name of the ImageStream to be tagged
required: false
value: ""
- name: CONTAINER_REGISTRY_CREDENTIALS
displayName: Secret name of container registries used for pulling and pushing images
value: factory2-pipeline-registry-credentials
required: false
- name: MESSAGING_PROVIDER
displayName: Name of the JMS messaging provider
value: Red Hat UMB
@@ -47,6 +56,11 @@ parameters:
- name: JENKINS_AGENT_CLOUD_NAME
displayName: Name of OpenShift cloud in Jenkins master configuration
value: openshift
- name: PROMOTING_DESTINATIONS
displayName: Comma seperated list of container repositories (without tags) to which the image will be promoted
description: OpenShift registries must be prefixed with 'atomic:'
required: true
{% include "snippets/c3i-library-parameters.yaml" %}
objects:
- kind: ServiceAccount
apiVersion: v1
@@ -74,14 +88,12 @@ objects:
spec:
runPolicy: "Parallel"
completionDeadlineSeconds: 1800
source:
git:
uri: "${MBS_GIT_REPO}"
ref: "${MBS_GIT_REF}"
strategy:
type: JenkinsPipeline
jenkinsPipelineStrategy:
env:
- name: PROMOTING_DESTINATIONS
value: "${PROMOTING_DESTINATIONS}"
- name: JENKINS_AGENT_CLOUD_NAME
value: "${JENKINS_AGENT_CLOUD_NAME}"
- name: JENKINS_AGENT_IMAGE
@@ -90,10 +102,14 @@ objects:
value: "${NAME}-jenkins-slave"
- name: SOURCE_CONTAINER_REPO
value: "${SOURCE_CONTAINER_REPO}"
- name: CONTAINER_REGISTRY_CREDENTIALS
value: "${CONTAINER_REGISTRY_CREDENTIALS}"
- name: TARGET_TAG
value: "${TARGET_TAG}"
- name: IMAGE_PROMOTION_JOB
value: "${IMAGE_PROMOTION_JOB}"
- name: TAG_INTO_IMAGESTREAM
value: "${TAG_INTO_IMAGESTREAM}"
- name: DEST_IMAGESTREAM_NAME
value: "${DEST_IMAGESTREAM_NAME}"
- name: DECISION_CONTEXT_REGEX
value: "${DECISION_CONTEXT_REGEX}"
- name: SUBJECT_IDENTIFIER_REGEX
@@ -107,4 +123,5 @@ objects:
value:
- name: MESSAGE_HEADERS
value:
jenkinsfilePath: openshift/integration/koji/pipelines/templates/mbs-greenwave-trigger.Jenkinsfile
jenkinsfile: |
{% filter indent(width=10) %}{% include "mbs-greenwave-trigger.Jenkinsfile" %}{% endfilter %}

120
openshift/integration/koji/pipelines/templates/mbs-image-promotion-template.yaml
View File

@@ -1,120 +0,0 @@
# Template to produce a pipeline for promoting images between environments
#
# The pipeline pulls the image to be promoted, then pushes it to destinations with promoted tags.
# Optionally, it tags the promoted image into an image stream after pushes.
---
apiVersion: v1
kind: Template
metadata:
name: mbs-image-promotion
labels:
template: mbs-image-promotion
parameters:
- name: NAME
displayName: Short unique identifier for the templated instances
description: This field is used to deploy multiple pipelines to one OpenShift project from this template.
required: true
- name: MBS_GIT_REPO
displayName: MBS Git repo URL
description: Default MBS Git repo URL in which to run functional tests against
required: true
value: "https://pagure.io/fm-orchestrator.git"
- name: MBS_GIT_REF
displayName: MBS Git repo ref
description: Default MBS Git repo ref in which to run functional tests against
required: true
value: master
- name: IMAGE
displayName: The container image to be promoted
description: This field must be in repo:tag or repo@sha256 format
value: provided-by-trigger
- name: PROMOTING_DESTINATIONS
displayName: Comma seperated list of container repositories (without tags) to which the image will be promoted
description: OpenShift registries must be prefixed with 'atomic:'
required: true
- name: CONTAINER_REGISTRY_CREDENTIALS
displayName: Secret name of container registries used for pulling and pushing images
value: factory2-pipeline-registry-credentials
required: false
- name: TAG_INTO_IMAGESTREAM
displayName: Whether to tag the image into an ImageStream
value: "false"
required: true
- name: DEST_IMAGESTREAM_NAME
displayName: Name of the ImageStream to be tagged
required: false
value: ""
- name: DEST_IMAGESTREAM_NAMESPACE
displayName: Namespace of the ImageStream to be tagged
description: Leaving blank means using the same namespace as the pipeline build
required: false
value: ""
- name: DEST_TAG
displayName: Name of the new tag
required: true
- name: JENKINS_AGENT_IMAGE
displayName: Container image for Jenkins slave pods
required: true
value: quay.io/factory2/mbs-jenkins-slave:latest
- name: JENKINS_AGENT_CLOUD_NAME
displayName: Name of OpenShift cloud in Jenkins master configuration
required: true
value: openshift
objects:
- kind: ServiceAccount
apiVersion: v1
metadata:
name: "${NAME}-jenkins-slave"
labels:
app: "${NAME}"
- kind: RoleBinding
apiVersion: v1
metadata:
name: "${NAME}-jenkins-slave_edit"
labels:
app: "${NAME}"
subjects:
- kind: ServiceAccount
name: "${NAME}-jenkins-slave"
roleRef:
name: edit
- kind: "BuildConfig"
apiVersion: "v1"
metadata:
name: "${NAME}"
labels:
app: "${NAME}"
spec:
runPolicy: "Parallel"
completionDeadlineSeconds: 1800
source:
git:
uri: "${MBS_GIT_REPO}"
ref: "${MBS_GIT_REF}"
strategy:
type: JenkinsPipeline
source:
type: None
jenkinsPipelineStrategy:
env:
- name: IMAGE
value: "${IMAGE}"
- name: PROMOTING_DESTINATIONS
value: "${PROMOTING_DESTINATIONS}"
- name: CONTAINER_REGISTRY_CREDENTIALS
value: "${CONTAINER_REGISTRY_CREDENTIALS}"
- name: TAG_INTO_IMAGESTREAM
value: "${TAG_INTO_IMAGESTREAM}"
- name: DEST_IMAGESTREAM_NAME
value: "${DEST_IMAGESTREAM_NAME}"
- name: DEST_IMAGESTREAM_NAMESPACE
value: "${DEST_IMAGESTREAM_NAMESPACE}"
- name: DEST_TAG
value: "${DEST_TAG}"
- name: JENKINS_AGENT_IMAGE
value: "${JENKINS_AGENT_IMAGE}"
- name: JENKINS_AGENT_CLOUD_NAME
value: "${JENKINS_AGENT_CLOUD_NAME}"
- name: JENKINS_AGENT_SERVICE_ACCOUNT
value: "${NAME}-jenkins-slave"
jenkinsfilePath: openshift/integration/koji/pipelines/templates/mbs-image-promotion.Jenkinsfile

125
openshift/integration/koji/pipelines/templates/mbs-image-promotion.Jenkinsfile
View File

@@ -1,125 +0,0 @@
pipeline {
agent {
kubernetes {
cloud "${params.JENKINS_AGENT_CLOUD_NAME}"
label "jenkins-slave-${UUID.randomUUID().toString()}"
serviceAccount "${params.JENKINS_AGENT_SERVICE_ACCOUNT}"
defaultContainer 'jnlp'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
app: "jenkins-${env.JOB_BASE_NAME.take(50)}"
factory2-pipeline-kind: "mbs-image-promotion-pipeline"
factory2-pipeline-build-number: "${env.BUILD_NUMBER}"
spec:
containers:
- name: jnlp
image: "${params.JENKINS_AGENT_IMAGE}"
imagePullPolicy: Always
tty: true
env:
- name: REGISTRY_CREDENTIALS
valueFrom:
secretKeyRef:
name: "${params.CONTAINER_REGISTRY_CREDENTIALS}"
key: '.dockerconfigjson'
resources:
requests:
memory: 512Mi
cpu: 300m
limits:
memory: 768Mi
cpu: 500m
"""
}
}
options {
timestamps()
timeout(time: 30, unit: 'MINUTES')
buildDiscarder(logRotator(numToKeepStr: '10'))
skipDefaultCheckout()
}
environment {
PIPELINE_NAMESPACE = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/namespace').trim()
SERVICE_ACCOUNT_TOKEN = readFile(file: '/run/secrets/kubernetes.io/serviceaccount/token').trim()
}
stages {
stage ('Prepare') {
steps {
script {
// Setting up registry credentials
dir ("${env.HOME}/.docker") {
// for the OpenShift internal registry
def dockerConfig = readJSON text: '{ "auths": {} }'
dockerConfig.auths['docker-registry.default.svc:5000'] = [
'email': '',
'auth': sh(returnStdout: true, script: 'set +x; echo -n "serviceaccount:$SERVICE_ACCOUNT_TOKEN" | base64 -').trim()
]
// merging user specified credentials
if (env.REGISTRY_CREDENTIALS) {
toBeMerged = readJSON text: env.REGISTRY_CREDENTIALS
dockerConfig.auths.putAll(toBeMerged.auths)
}
// writing to ~/.docker/config.json
writeJSON file: 'config.json', json: dockerConfig
}
}
}
}
stage('Pull image') {
steps {
echo "Pulling container image ${params.IMAGE}..."
withEnv(["SOURCE_IMAGE_REF=${params.IMAGE}"]) {
sh '''
set -e +x # hide the token from Jenkins console
mkdir -p _image
skopeo copy docker://"$SOURCE_IMAGE_REF" dir:_image
'''
}
}
}
stage('Promote image') {
steps {
script {
def destinations = params.PROMOTING_DESTINATIONS ? params.PROMOTING_DESTINATIONS.split(',') : []
openshift.withCluster() {
def pushTasks = destinations.collectEntries {
["Pushing ${it}" : {
def dest = "${it}:${params.DEST_TAG}"
// Only docker and atomic registries are allowed
if (!dest.startsWith('atomic:') && !dest.startsWith('docker://')) {
dest = "docker://${dest}"
}
echo "Pushing container image to ${dest}..."
withEnv(["DEST_IMAGE_REF=${dest}"]) {
retry(5) {
sh 'skopeo copy dir:_image "$DEST_IMAGE_REF"'
}
}
}]
}
parallel pushTasks
}
}
}
}
stage('Tag ImageStream') {
when {
expression {
return params.DEST_IMAGESTREAM_NAME && params.TAG_INTO_IMAGESTREAM == "true"
}
}
steps {
script {
def destRef = "${params.DEST_IMAGESTREAM_NAMESPACE ?: env.PIPELINE_NAMESPACE}/${params.DEST_IMAGESTREAM_NAME}:${params.DEST_TAG}"
openshift.withCluster() {
echo "Tagging ${params.IMAGE} into ${destRef}..."
openshift.tag('--source=docker', params.IMAGE, destRef)
}
}
}
}
}
}

143
openshift/integration/koji/pipelines/templates/mbs-integration-test-template.yaml
View File

@@ -1,143 +0,0 @@
# Template to produce a new OpenShift pipeline for running integration tests
#
---
apiVersion: v1
kind: Template
metadata:
name: mbs-integration-test
labels:
template: mbs-integration-test
parameters:
- name: NAME
displayName: Short unique identifier for the templated instances
description: This field is used to deploy multiple pipelines to one OpenShift project from this template.
required: true
value: mbs-integration-test
- name: MBS_BACKEND_IMAGE
displayName: The MBS backend container image to be tested
description: This field must be in repo:tag or repo@sha256 format
value: quay.io/factory2/mbs-backend:latest
- name: MBS_FRONTEND_IMAGE
displayName: The MBS frontend container image to be tested
description: This field must be in repo:tag or repo@sha256 format
value: quay.io/factory2/mbs-frontend:latest
- name: MBS_GIT_REPO
displayName: MBS Git repo URL
description: Default MBS Git repo URL in which to find the integration tests to run
required: true
value: "https://pagure.io/fm-orchestrator.git"
- name: MBS_GIT_REF
displayName: MBS Git repo ref
description: Default MBS Git repo ref in which to find the integration tests to run
required: true
value: master
- name: KOJI_IMAGE
displayName: The Koji container image to be tested
description: This field must be in repo:tag or repo@sha256 format
value: quay.io/factory2/koji:latest
- name: TEST_IMAGES
displayName: Images being tested
description: >-
A space-separated list of the refs of the images being tested.
Results of the tests will be reported to ResultsDB.
- name: JENKINS_AGENT_IMAGE
displayName: Container image for Jenkins slave pods
required: true
value: quay.io/factory2/mbs-jenkins-slave:latest
- name: JENKINS_AGENT_CLOUD_NAME
displayName: Name of OpenShift cloud in Jenkins master configuration
required: true
value: openshift
- name: TEST_NAMESPACE
displayName: >-
Namespace where the Jenkins agent for this test will run, and where
test resources will be created.
required: false
- name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE
displayName: Namespace with pipeline-as-a-service build
value: c3i
- name: ENVIRONMENT
displayName: environment name (dev/stage/prod)
required: true
value: dev
- name: MESSAGING_PROVIDER
displayName: Name of the JMS messaging provider
value: Red Hat UMB
- name: TESTCASES
displayName: >-
Space-separated list of testcases to run as part of the pipeline. An empty string (the default)
causes all available testcases to run. The value "skip" causes no testcases to be run.
required: false
value: ""
- name: CLEANUP
displayName: Cleanup objects after testing is complete
required: true
value: "true"
objects:
- kind: ServiceAccount
apiVersion: v1
metadata:
name: "${NAME}-jenkins-slave"
labels:
app: "${NAME}"
- kind: RoleBinding
apiVersion: v1
metadata:
name: "${NAME}-jenkins-slave_edit"
labels:
app: "${NAME}"
subjects:
- kind: ServiceAccount
name: "${NAME}-jenkins-slave"
roleRef:
name: edit
- kind: "BuildConfig"
apiVersion: "v1"
metadata:
name: "${NAME}"
labels:
app: "${NAME}"
spec:
runPolicy: "Parallel"
completionDeadlineSeconds: 1800
source:
git:
uri: "${MBS_GIT_REPO}"
ref: "${MBS_GIT_REF}"
strategy:
type: JenkinsPipeline
jenkinsPipelineStrategy:
env:
- name: MBS_GIT_REPO
value: "${MBS_GIT_REPO}"
- name: MBS_GIT_REF
value: "${MBS_GIT_REF}"
- name: MBS_BACKEND_IMAGE
value: "${MBS_BACKEND_IMAGE}"
- name: MBS_FRONTEND_IMAGE
value: "${MBS_FRONTEND_IMAGE}"
- name: KOJI_IMAGE
value: "${KOJI_IMAGE}"
- name: TEST_IMAGES
value: "${TEST_IMAGES}"
- name: IMAGE_IS_SCRATCH
value: "true"
- name: JENKINS_AGENT_IMAGE
value: "${JENKINS_AGENT_IMAGE}"
- name: JENKINS_AGENT_CLOUD_NAME
value: "${JENKINS_AGENT_CLOUD_NAME}"
- name: TEST_NAMESPACE
value: "${TEST_NAMESPACE}"
- name: ENVIRONMENT
value: "${ENVIRONMENT}"
- name: MESSAGING_PROVIDER
value: "${MESSAGING_PROVIDER}"
- name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE
value: "${PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE}"
- name: JENKINS_AGENT_SERVICE_ACCOUNT
value: "${NAME}-jenkins-slave"
- name: TESTCASES
value: "${TESTCASES}"
- name: CLEANUP
value: "${CLEANUP}"
jenkinsfilePath: openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile

267
openshift/integration/koji/pipelines/templates/mbs-integration-test.Jenkinsfile
View File

@@ -1,267 +0,0 @@
library identifier: 'c3i@master', changelog: false,
retriever: modernSCM([$class: 'GitSCMSource', remote: 'https://pagure.io/c3i-library.git'])
def deployments
pipeline {
agent {
kubernetes {
cloud params.JENKINS_AGENT_CLOUD_NAME
label "jenkins-slave-${UUID.randomUUID().toString()}"
serviceAccount params.JENKINS_AGENT_SERVICE_ACCOUNT
defaultContainer 'jnlp'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
app: "jenkins-${env.JOB_BASE_NAME.take(50)}"
factory2-pipeline-kind: "mbs-integration-test-pipeline"
factory2-pipeline-build-number: "${env.BUILD_NUMBER}"
spec:
containers:
- name: jnlp
image: "${params.JENKINS_AGENT_IMAGE}"
imagePullPolicy: Always
tty: true
resources:
requests:
memory: 512Mi
cpu: 300m
limits:
memory: 768Mi
cpu: 500m
"""
}
}
options {
timestamps()
timeout(time: 60, unit: 'MINUTES')
buildDiscarder(logRotator(numToKeepStr: '10'))
skipDefaultCheckout()
}
environment {
PIPELINE_ID = "${params.TEST_NAMESPACE}"
}
stages {
stage('Prepare') {
steps {
script {
// MBS_GIT_REF can be either a regular branch (in the heads/ namespace), a pull request
// branch (in the pull/ namespace), or a full 40-character sha1, which is assumed to
// exist on the master branch.
def branch = params.MBS_GIT_REF ==~ '[0-9a-f]{40}' ? 'master' : params.MBS_GIT_REF
c3i.clone(repo: params.MBS_GIT_REPO, branch: branch, rev: params.MBS_GIT_REF)
// get current commit ID
// FIXME: Due to a bug discribed in https://issues.jenkins-ci.org/browse/JENKINS-45489,
// the return value of checkout() is unreliable.
// Not working: env.MBS_GIT_COMMIT = scmVars.GIT_COMMIT
env.MBS_GIT_COMMIT = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
echo "Running integration tests for ${branch}, commit=${env.MBS_GIT_COMMIT}"
currentBuild.displayName = "${branch}: ${env.MBS_GIT_COMMIT.take(7)}"
}
}
}
stage('Cleanup') {
when {
expression {
// Only run cleanup if we're running tests in the default namespace.
return !params.TEST_NAMESPACE
}
}
steps {
script {
openshift.withCluster() {
openshift.withProject() {
// Cleanup all test environments that were created 1 hour ago in case of failures of previous cleanups.
c3i.cleanup(script: this, 'krb5', 'umb', 'koji', 'mbs')
}
}
}
}
post {
failure {
echo "Cleanup of old environments FAILED"
}
}
}
stage('Route suffix') {
when {
expression { !env.PAAS_DOMAIN }
}
steps {
script {
openshift.withCluster() {
openshift.withProject(env.PIPELINE_ID) {
def testroute = openshift.create('route', 'edge', 'test', '--service=test', '--port=8080')
def testhost = testroute.object().spec.host
env.PAAS_DOMAIN = testhost.minus("test-${env.PIPELINE_ID}.")
testroute.delete()
}
}
}
}
post {
success {
echo "Routes end with ${env.PAAS_DOMAIN}"
}
}
}
stage('Deploy test environment') {
steps {
script {
openshift.withCluster() {
openshift.withProject(params.PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE) {
c3i.buildAndWait(script: this, objs: "bc/pipeline-as-a-service",
'-e', "DEFAULT_IMAGE_TAG=${env.ENVIRONMENT}",
'-e', "PIPELINE_ID=${env.PIPELINE_ID}",
'-e', "WAIVERDB_IMAGE=",
'-e', "C3IAAS_PROJECT=",
'-e', "RESULTSDB_IMAGE=",
'-e', "RESULTSDB_UPDATER_IMAGE=",
'-e', "GREENWAVE_IMAGE=",
'-e', "DATAGREPPER_IMAGE=",
'-e', "DATANOMMER_IMAGE=",
'-e', "MBS_BACKEND_IMAGE=${env.MBS_BACKEND_IMAGE}",
'-e', "MBS_FRONTEND_IMAGE=${env.MBS_FRONTEND_IMAGE}",
'-e', "PAAS_DOMAIN=${env.PAAS_DOMAIN}"
)
}
}
}
}
}
stage('Run tests') {
steps {
script {
def testcases
if (params.TESTCASES) {
if (params.TESTCASES == 'skip') {
testcases = []
echo 'Skipping integration tests'
} else {
testcases = params.TESTCASES.split()
echo "Using specified list of testcases: ${testcases}"
}
} else {
testcases = findFiles(glob: 'openshift/integration/koji/pipelines/tests/*.groovy').collect {
it.name.minus('.groovy')
}
echo "Using all available testcases: ${testcases}"
}
testcases.each { testcase ->
env.CURRENT_TESTCASE = testcase
echo "Running testcase ${testcase}..."
def test = load "openshift/integration/koji/pipelines/tests/${testcase}.groovy"
test.runTests()
}
}
}
post {
success {
echo "All tests successful"
}
failure {
echo "Testcase ${env.CURRENT_TESTCASE} FAILED"
}
}
}
}
post {
success {
script {
params.TEST_IMAGES.split(',').each {
sendToResultsDB(it, 'passed')
}
}
}
failure {
script {
params.TEST_IMAGES.split(',').each {
sendToResultsDB(it, 'failed')
}
openshift.withCluster() {
openshift.withProject(params.TEST_NAMESPACE) {
echo 'Getting logs from all deployments...'
openshift.selector('pods', ['c3i.redhat.com/pipeline': env.PIPELINE_ID]).logs('--tail 100')
}
}
}
}
cleanup {
script {
if (params.CLEANUP == 'true' && !params.TEST_NAMESPACE) {
openshift.withCluster() {
/* Tear down everything we just created */
echo 'Tearing down test resources...'
openshift.selector('all,pvc,configmap,secret',
['c3i.redhat.com/pipeline': env.PIPELINE_ID]).delete('--ignore-not-found=true')
}
} else {
echo 'Skipping cleanup'
}
}
}
}
}
def sendToResultsDB(imageRef, status) {
if (!params.MESSAGING_PROVIDER) {
echo "Message bus is not set. Skipping send of:\nimageRef: ${imageRef}\nstatus: ${status}"
return
}
def (repourl, digest) = imageRef.tokenize('@')
def (registry, reponame) = repourl.split('/', 2)
def image = reponame.split('/').last()
def sendResult = sendCIMessage \
providerName: params.MESSAGING_PROVIDER, \
overrides: [topic: 'VirtualTopic.eng.ci.container-image.test.complete'], \
messageType: 'Custom', \
messageProperties: '', \
messageContent: """
{
"ci": {
"name": "C3I Jenkins",
"team": "DevOps",
"url": "${env.JENKINS_URL}",
"docs": "https://pagure.io/fm-orchestrator/blob/master/f/openshift/integration/koji",
"irc": "#pnt-devops-dev",
"email": "pnt-factory2-devel@redhat.com",
"environment": "${params.ENVIRONMENT}"
},
"run": {
"url": "${env.BUILD_URL}",
"log": "${env.BUILD_URL}/console",
"debug": "",
"rebuild": "${env.BUILD_URL}/rebuild/parametrized"
},
"artifact": {
"type": "container-image",
"repository": "${reponame}",
"digest": "${digest}",
"nvr": "${imageRef}",
"issuer": "c3i-jenkins",
"scratch": ${params.IMAGE_IS_SCRATCH},
"id": "${image}@${digest}"
},
"system":
[{
"os": "${params.JENKINS_AGENT_IMAGE}",
"provider": "openshift",
"architecture": "x86_64"
}],
"type": "integration",
"category": "${params.ENVIRONMENT}",
"status": "${status}",
"xunit": "",
"generated_at": "${new Date().format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone('UTC'))}",
"namespace": "c3i",
"version": "0.1.0"
}
"""
if (sendResult.getMessageId()) {
// echo sent message id and content
echo 'Successfully sent the test result to ResultsDB.'
echo "Message ID: ${sendResult.getMessageId()}"
echo "Message content: ${sendResult.getMessageContent()}"
} else {
echo 'Failed to sent the test result to ResultsDB.'
}
}

103
openshift/integration/koji/pipelines/templates/mbs-polling-pagure.Jenkinsfile Normal file
View File

@@ -0,0 +1,103 @@
{% include "snippets/c3i-library.groovy" %}
pipeline {
{% include "snippets/default-agent.groovy" %}
options {
timestamps()
timeout(time: 60, unit: 'MINUTES')
buildDiscarder(logRotator(numToKeepStr: '10'))
}
environment {
PIPELINE_NAMESPACE = readFile('/run/secrets/kubernetes.io/serviceaccount/namespace').trim()
PAGURE_URL = "${PAGURE_URL}"
PAGURE_API = "${env.PAGURE_URL}/api/0"
PAGURE_REPO_NAME = "${PAGURE_REPO_NAME}"
PAGURE_REPO_IS_FORK = "${PAGURE_REPO_IS_FORK}"
PAGURE_POLLING_FOR_PR = "${PAGURE_POLLING_FOR_PR}"
PAGURE_REPO_HOME = "${env.PAGURE_URL}${env.PAGURE_REPO_IS_FORK == 'true' ? '/fork' : ''}/${env.PAGURE_REPO_NAME}"
GIT_URL = "${env.PAGURE_URL}/${env.PAGURE_REPO_IS_FORK == 'true' ? 'forks/' : ''}${env.PAGURE_REPO_NAME}.git"
PREMERGE_JOB_NAME = "${PREMERGE_JOB_NAME}"
POSTMERGE_JOB_NAME = "${POSTMERGE_JOB_NAME}"
}
triggers { pollSCM("${PAGURE_POLLING_SCHEDULE}") }
stages {
stage('Prepare') {
agent { label 'master' }
steps {
script {
def polled = env.PAGURE_POLLING_FOR_PR == 'true' ? 'pull/*/head' : "${PAGURE_POLLED_BRANCH}"
// Need to prefix the rev with origin/ for pollSCM to work correctly
def rev = "origin/${polled}"
def scmVars = c3i.clone(repo: env.GIT_URL, branch: polled, rev: rev)
env.GIT_COMMIT = scmVars.GIT_COMMIT
// setting build display name
def prefix = 'origin/'
def branch = scmVars.GIT_BRANCH.startsWith(prefix) ? scmVars.GIT_BRANCH.substring(prefix.size())
: scmVars.GIT_BRANCH // origin/pull/1234/head -> pull/1234/head, origin/master -> master
env.MBS_GIT_BRANCH = branch
echo "Build on branch=${env.MBS_GIT_BRANCH}, commit=${env.GIT_COMMIT}"
if (env.PAGURE_POLLING_FOR_PR == 'false') {
currentBuild.displayName = "${env.MBS_GIT_BRANCH}: ${env.GIT_COMMIT.substring(0, 7)}"
currentBuild.description = """<a href="${env.PAGURE_REPO_HOME}/c/${env.GIT_COMMIT}">${currentBuild.displayName}</a>"""
} else if (env.PAGURE_POLLING_FOR_PR == 'true' && branch ==~ /^pull\/[0-9]+\/head$/) {
env.PR_NO = branch.split('/')[1]
def prInfo = pagure.getPR(env.PR_NO)
if (prInfo.status == 'Open') {
env.PR_URL = "${env.PAGURE_REPO_HOME}/pull-request/${env.PR_NO}"
// To HTML syntax in build description, go to `Jenkins/Global Security/Markup Formatter` and select 'Safe HTML'.
def pagureLink = """<a href="${env.PR_URL}">PR#${env.PR_NO}</a>"""
echo "Building PR #${env.PR_NO}: ${env.PR_URL}"
currentBuild.displayName = "PR#${env.PR_NO}"
currentBuild.description = pagureLink
} else {
echo "Skipping PR#${env.PR_NO} because it is ${prInfo.status}"
env.SKIP = 'true'
}
} else { // This shouldn't happen.
error("Build is aborted due to unexpected polling trigger actions.")
}
}
}
}
stage('Update pipeline jobs') {
when {
expression {
return "${PIPELINE_UPDATE_JOBS_DIR}" && env.PAGURE_POLLING_FOR_PR == 'false' && env.MBS_GIT_BRANCH == "${PAGURE_POLLED_BRANCH}"
}
}
steps {
script {
c3i.clone(repo: env.GIT_URL, branch: env.MBS_GIT_BRANCH)
dir('openshift/integration/koji/pipelines') {
sh '''
make install JOBS_DIR="${PIPELINE_UPDATE_JOBS_DIR}"
'''
}
}
}
}
stage('Build') {
when {
not {
environment name: 'SKIP', value: 'true'
}
}
steps {
script {
openshift.withCluster() {
echo 'Starting a MBS build run...'
def devBuild = c3i.build(script: this,
objs: "bc/${env.PAGURE_POLLING_FOR_PR == 'true' ? env.PREMERGE_JOB_NAME : env.POSTMERGE_JOB_NAME}",
'-e', "MBS_GIT_REF=${env.MBS_GIT_BRANCH}", '-e', "PAGURE_REPO_IS_FORK=${env.PAGURE_REPO_IS_FORK}",
'-e', "PAGURE_REPO_NAME=${env.PAGURE_REPO_NAME}"
)
c3i.waitForBuildStart(script: this, build: devBuild)
def devBuildInfo = devBuild.object()
def downstreamBuildName = devBuildInfo.metadata.name
def downstreamBuildUrl = devBuildInfo.metadata.annotations['openshift.io/jenkins-build-uri']
echo "Downstream build ${downstreamBuildName}(${downstreamBuildUrl}) started."
}
}
}
}
}
}

144
openshift/integration/koji/pipelines/templates/mbs-polling-pagure.yaml
View File

@@ -58,6 +58,7 @@ parameters:
displayName: Name of OpenShift cloud in Jenkins master configuration
required: true
value: openshift
{% include "snippets/c3i-library-parameters.yaml" %}
objects:
- kind: ServiceAccount
apiVersion: v1
@@ -88,139 +89,12 @@ objects:
strategy:
type: JenkinsPipeline
jenkinsPipelineStrategy:
env:
- name: JENKINS_AGENT_CLOUD_NAME
value: "${JENKINS_AGENT_CLOUD_NAME}"
- name: JENKINS_AGENT_IMAGE
value: "${JENKINS_AGENT_IMAGE}"
- name: JENKINS_AGENT_SERVICE_ACCOUNT
value: "${NAME}-jenkins-slave"
jenkinsfile: |-
// Don't use external Jenkinsfile here, or Jenkins will also poll on that repo and branch
library identifier: 'c3i@master', changelog: false,
retriever: modernSCM([$class: 'GitSCMSource', remote: 'https://pagure.io/c3i-library.git'])
pipeline {
agent {
kubernetes {
cloud "${JENKINS_AGENT_CLOUD_NAME}"
label "jenkins-slave-${UUID.randomUUID().toString()}"
serviceAccount "${NAME}-jenkins-slave"
defaultContainer 'jnlp'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
app: "jenkins-${env.JOB_BASE_NAME.take(50)}"
factory2-pipeline-kind: "mbs-polling-to-pagure-pipeline"
factory2-pipeline-build-number: "${env.BUILD_NUMBER}"
spec:
containers:
- name: jnlp
image: "${JENKINS_AGENT_IMAGE}"
imagePullPolicy: Always
tty: true
resources:
requests:
memory: 512Mi
cpu: 300m
limits:
memory: 768Mi
cpu: 500m
"""
}
}
options {
timestamps()
timeout(time: 60, unit: 'MINUTES')
buildDiscarder(logRotator(numToKeepStr: '10'))
skipDefaultCheckout()
}
environment {
PIPELINE_NAMESPACE = readFile('/run/secrets/kubernetes.io/serviceaccount/namespace').trim()
PAGURE_URL = "${PAGURE_URL}"
PAGURE_API = "${env.PAGURE_URL}/api/0"
PAGURE_REPO_NAME = "${PAGURE_REPO_NAME}"
PAGURE_REPO_IS_FORK = "${PAGURE_REPO_IS_FORK}"
PAGURE_POLLING_FOR_PR = "${PAGURE_POLLING_FOR_PR}"
PAGURE_REPO_HOME = "${env.PAGURE_URL}${env.PAGURE_REPO_IS_FORK == 'true' ? '/fork' : ''}/${env.PAGURE_REPO_NAME}"
GIT_URL = "${env.PAGURE_URL}/${env.PAGURE_REPO_IS_FORK == 'true' ? 'forks/' : ''}${env.PAGURE_REPO_NAME}.git"
PREMERGE_JOB_NAME = "${PREMERGE_JOB_NAME}"
POSTMERGE_JOB_NAME = "${POSTMERGE_JOB_NAME}"
}
triggers { pollSCM("${PAGURE_POLLING_SCHEDULE}") }
stages {
stage('Prepare') {
agent { label 'master' }
steps {
script {
def polled = env.PAGURE_POLLING_FOR_PR == 'true' ? 'pull/*/head' : "${PAGURE_POLLED_BRANCH}"
// Need to prefix the rev with origin/ for pollSCM to work correctly
def rev = "origin/${polled}"
def scmVars = c3i.clone(repo: env.GIT_URL, branch: polled, rev: rev)
env.GIT_COMMIT = scmVars.GIT_COMMIT
// setting build display name
def prefix = 'origin/'
def branch = scmVars.GIT_BRANCH.startsWith(prefix) ? scmVars.GIT_BRANCH.substring(prefix.size())
: scmVars.GIT_BRANCH // origin/pull/1234/head -> pull/1234/head, origin/master -> master
env.MBS_GIT_BRANCH = branch
echo "Build on branch=${env.MBS_GIT_BRANCH}, commit=${env.GIT_COMMIT}"
if (env.PAGURE_POLLING_FOR_PR == 'false') {
currentBuild.displayName = "${env.MBS_GIT_BRANCH}: ${env.GIT_COMMIT.substring(0, 7)}"
currentBuild.description = """<a href="${env.PAGURE_REPO_HOME}/c/${env.GIT_COMMIT}">${currentBuild.displayName}</a>"""
} else if (env.PAGURE_POLLING_FOR_PR == 'true' && branch ==~ /^pull\/[0-9]+\/head$/) {
env.PR_NO = branch.split('/')[1]
def prInfo = pagure.getPR(env.PR_NO)
if (prInfo.status == 'Open') {
env.PR_URL = "${env.PAGURE_REPO_HOME}/pull-request/${env.PR_NO}"
// To HTML syntax in build description, go to `Jenkins/Global Security/Markup Formatter` and select 'Safe HTML'.
def pagureLink = """<a href="${env.PR_URL}">PR#${env.PR_NO}</a>"""
echo "Building PR #${env.PR_NO}: ${env.PR_URL}"
currentBuild.displayName = "PR#${env.PR_NO}"
currentBuild.description = pagureLink
} else {
echo "Skipping PR#${env.PR_NO} because it is ${prInfo.status}"
env.SKIP = 'true'
}
} else { // This shouldn't happen.
error("Build is aborted due to unexpected polling trigger actions.")
}
}
}
}
stage('Update pipeline jobs') {
when {
expression {
return "${PIPELINE_UPDATE_JOBS_DIR}" && env.PAGURE_POLLING_FOR_PR == 'false' && env.MBS_GIT_BRANCH == "${PAGURE_POLLED_BRANCH}"
}
}
steps {
script {
c3i.clone(repo: env.GIT_URL, branch: env.MBS_GIT_BRANCH)
dir('openshift/integration/koji/pipelines') {
sh '''
make install JOBS_DIR="${PIPELINE_UPDATE_JOBS_DIR}"
'''
}
}
}
}
stage('Build') {
when {
not {
environment name: 'SKIP', value: 'true'
}
}
steps {
script {
openshift.withCluster() {
echo 'Starting a MBS build run...'
def devBuild = c3i.build(script: this,
objs: "bc/${env.PAGURE_POLLING_FOR_PR == 'true' ? env.PREMERGE_JOB_NAME : env.POSTMERGE_JOB_NAME}",
'-e', "MBS_GIT_REF=${env.MBS_GIT_BRANCH}", '-e', "PAGURE_REPO_IS_FORK=${env.PAGURE_REPO_IS_FORK}",
'-e', "PAGURE_REPO_NAME=${env.PAGURE_REPO_NAME}"
)
c3i.waitForBuildStart(script: this, build: devBuild)
def devBuildInfo = devBuild.object()
def downstreamBuildName = devBuildInfo.metadata.name
def downstreamBuildUrl = devBuildInfo.metadata.annotations['openshift.io/jenkins-build-uri']
echo "Downstream build ${downstreamBuildName}(${downstreamBuildUrl}) started."
}
}
}
}
}
}
{% filter indent(width=10) %}{% include "mbs-polling-pagure.Jenkinsfile" %}{% endfilter %}

156
openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.Jenkinsfile
View File

@@ -1,110 +1,56 @@
// Use scripted syntax because CIBuildTrigger currently doesn't support the declarative syntax
properties([
buildDiscarder(logRotator(numToKeepStr: '10')),
pipelineTriggers([
// example: https://github.com/jenkinsci/jms-messaging-plugin/blob/9b9387c3a52f037ba0d019c2ebcf2a2796fc6397/src/test/java/com/redhat/jenkins/plugins/ci/integration/AmqMessagingPluginIntegrationTest.java
[$class: 'CIBuildTrigger',
providerData: [$class: 'ActiveMQSubscriberProviderData',
name: params.MESSAGING_PROVIDER,
overrides: [topic: params.MESSAGING_TOPIC],
selector: "repo = '${params.TRACKED_CONTAINER_REPO}' AND action IN ('added', 'updated') AND tag = '${params.TRACKED_TAG}'",
],
],
]),
])
if (!params.CI_MESSAGE) {
echo 'This build is not started by a CI message. Only configurations were done.'
return
}
library identifier: 'c3i@master', changelog: false,
retriever: modernSCM([$class: 'GitSCMSource', remote: 'https://pagure.io/c3i-library.git'])
def label = "jenkins-slave-${UUID.randomUUID().toString()}"
podTemplate(
cloud: "${params.JENKINS_AGENT_CLOUD_NAME}",
label: label,
serviceAccount: "${env.JENKINS_AGENT_SERVICE_ACCOUNT}",
yaml: """
apiVersion: v1
kind: Pod
metadata:
labels:
app: "jenkins-${env.JOB_BASE_NAME.take(50)}"
factory2-pipeline-kind: "mbs-repotracker-trigger"
factory2-pipeline-build-number: "${env.BUILD_NUMBER}"
spec:
containers:
- name: jnlp
image: ${params.JENKINS_AGENT_IMAGE}
imagePullPolicy: Always
tty: true
env:
- name: REGISTRY_CREDENTIALS
valueFrom:
secretKeyRef:
name: "${params.CONTAINER_REGISTRY_CREDENTIALS}"
key: ".dockerconfigjson"
resources:
requests:
memory: 256Mi
cpu: 200m
limits:
memory: 512Mi
cpu: 300m
"""
) {
node(label) {
stage('Allocate C3IaaS project') {
if (!(params.USE_C3IAAS == 'true' &&
params.C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE &&
params.C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAME)) {
echo "Not configured to use C3IaaS"
return
}
env.PIPELINE_NAMESPACE = readFile("/run/secrets/kubernetes.io/serviceaccount/namespace").trim()
env.C3IAAS_NAMESPACE = "c3i-mbs-tr-${params.TRACKED_TAG}-${env.BUILD_NUMBER}"
echo "Requesting new OpenShift project ${env.C3IAAS_NAMESPACE}..."
openshift.withCluster() {
openshift.withProject(params.C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE) {
c3i.buildAndWait(script: this, objs: "bc/${params.C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAME}",
'-e', "PROJECT_NAME=${env.C3IAAS_NAMESPACE}",
'-e', "ADMIN_GROUPS=system:serviceaccounts:${env.PIPELINE_NAMESPACE}",
'-e', "LIFETIME_IN_MINUTES=${params.C3IAAS_LIFETIME}"
)
}
}
echo "Allocated project ${env.C3IAAS_NAMESPACE}"
}
stage('Trigger tests') {
def message = readJSON text: params.CI_MESSAGE
echo "Tag :${message.tag} is ${message.action} in ${message.repo}. New digest: ${message.digest}"
def frontendImage = "${message.repo}@${message.digest}"
// We have the digest of the current frontend image with this tag.
// Lookup the digest of the current backend image with the same tag.
if (env.REGISTRY_CREDENTIALS) {
dir ("${env.HOME}/.docker") {
writeFile file: 'config.json', text: env.REGISTRY_CREDENTIALS
}
}
def output = sh(script: "skopeo inspect docker://${params.MBS_BACKEND_REPO}:${message.tag}", returnStdout: true).trim()
def backendData = readJSON text: output
def backendImage = "${params.MBS_BACKEND_REPO}@${backendData.Digest}"
echo "Current mbs-backend image is: ${backendImage}"
echo "Triggering a job to test if ${frontendImage} and ${backendImage} meet all criteria of desired tag"
openshift.withCluster() {
openshift.withProject(params.TEST_JOB_NAMESPACE) {
def build = c3i.build(script: this, objs: "bc/${params.TEST_JOB_NAME}",
'-e', "MBS_BACKEND_IMAGE=${backendImage}",
'-e', "MBS_FRONTEND_IMAGE=${frontendImage}",
'-e', "TEST_IMAGES=${backendImage},${frontendImage}",
'-e', "IMAGE_IS_SCRATCH=false",
'-e', "TEST_NAMESPACE=${env.C3IAAS_NAMESPACE ?: ''}",
)
c3i.waitForBuildStart(script: this, build: build)
buildInfo = build.object()
echo "Build ${buildInfo.metadata.annotations['openshift.io/jenkins-build-uri'] ?: buildInfo.metadata.name} started."
{% include "snippets/c3i-library.groovy" %}
pipeline {
{% include "snippets/default-agent.groovy" %}
options {
timestamps()
timeout(time: 120, unit: 'MINUTES')
buildDiscarder(logRotator(numToKeepStr: '10'))
}
triggers {
ciBuildTrigger(
noSquash: false,
providerList: [
activeMQSubscriber(
name: params.MESSAGING_PROVIDER,
overrides: [topic: params.MESSAGING_TOPIC],
selector: "repo = '${params.TRACKED_CONTAINER_REPO}' AND action IN ('added', 'updated') AND tag = '${params.TRACKED_TAG}'",
)
]
)
}
stages {
stage("Message Check and setup") {
steps {
script {
if (!params.CI_MESSAGE) {
error("This build is not started by a CI message. Only configurations were done.")
}
c3i.clone(repo: params.MBS_GIT_REPO, branch: params.MBS_GIT_REF)
def message = readJSON text: params.CI_MESSAGE
echo "Tag :${message.tag} is ${message.action} in ${message.repo}. New digest: ${message.digest}"
env.FRONTEND_IMAGE_REF = "${message.repo}@${message.digest}"
// We have the digest of the current frontend image with this tag.
// Lookup the digest of the current backend image with the same tag.
if (params.CONTAINER_REGISTRY_CREDENTIALS) {
dir ("${env.HOME}/.docker") {
openshift.withCluster() {
def dockerconf = openshift.selector('secret', params.CONTAINER_REGISTRY_CREDENTIALS).object().data['.dockerconfigjson']
writeFile file: 'config.json', text: dockerconf, encoding: "Base64"
}
}
}
def output = sh(script: "skopeo inspect docker://${params.MBS_BACKEND_REPO}:${message.tag}", returnStdout: true).trim()
def backendData = readJSON text: output
env.BACKEND_IMAGE_REF = "${params.MBS_BACKEND_REPO}@${backendData.Digest}"
echo "Current mbs-backend image is: ${env.BACKEND_IMAGE_REF}"
echo "Triggering a job to test if ${env.FRONTEND_IMAGE_REF} and ${env.BACKEND_IMAGE_REF} meet all criteria of desired tag"
env.C3IAAS_PROJECT = params.C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE
env.IMAGE_IS_SCRATCH = false
env.PIPELINE_ID = "c3i-mbs-tag-${message.tag}-${message.digest[-9..-1]}"
}
}
}
{% include "snippets/mbs-integration-test.groovy" %}
}
}
{% include "snippets/functions.groovy" %}

36
openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.yaml
View File

@@ -15,10 +15,12 @@ parameters:
- name: MBS_GIT_REPO
displayName: MBS Git repo URL
description: Default MBS Git repo URL in which to run dev tests against
value: "https://pagure.io/fm-orchestrator.git"
required: true
value: https://pagure.io/fm-orchestrator.git
- name: MBS_GIT_REF
displayName: MBS Git repo ref
description: Default MBS Git repo ref in which to run dev tests against
required: true
value: master
- name: TRACKED_CONTAINER_REPO
displayName: Container repo to be tracked
@@ -45,12 +47,10 @@ parameters:
- name: MESSAGING_TOPIC
displayName: Name of the topic that the trigger subscribes to
value: "Consumer.rh-jenkins-ci-plugin.c3i-mbs-repotracker-trigger.VirtualTopic.eng.repotracker.container.tag.>"
- name: TEST_JOB_NAME
displayName: Name of integration test job to trigger
- name: ENVIRONMENT
displayName: environment name (dev/stage/prod)
required: true
- name: TEST_JOB_NAMESPACE
displayName: Namespace in which to trigger the integration test job
required: false
value: dev
- name: USE_C3IAAS
displayName: >-
Use C3I-as-a-Service to dynamically allocate a temporary OpenShift project for building
@@ -69,6 +69,11 @@ parameters:
displayName: The lifetime of the OpenShift project allocated by C3I-as-a-Service.
required: true
value: "120"
- name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE
displayName: The namespace where the Pipeline-as-a-Service project request BuildConfig has been defined
required: false
value: c3i
{% include "snippets/c3i-library-parameters.yaml" %}
objects:
- kind: ServiceAccount
apiVersion: v1
@@ -96,10 +101,6 @@ objects:
spec:
runPolicy: "Parallel"
completionDeadlineSeconds: 1800
source:
git:
uri: "${MBS_GIT_REPO}"
ref: "${MBS_GIT_REF}"
strategy:
type: JenkinsPipeline
jenkinsPipelineStrategy:
@@ -110,6 +111,10 @@ objects:
value: "${JENKINS_AGENT_IMAGE}"
- name: JENKINS_AGENT_SERVICE_ACCOUNT
value: "${NAME}-jenkins-slave"
- name: MBS_GIT_REPO
value: "${MBS_GIT_REPO}"
- name: MBS_GIT_REF
value: "${MBS_GIT_REF}"
- name: TRACKED_CONTAINER_REPO
value: "${TRACKED_CONTAINER_REPO}"
- name: TRACKED_TAG
@@ -118,18 +123,18 @@ objects:
value: "${MBS_BACKEND_REPO}"
- name: CONTAINER_REGISTRY_CREDENTIALS
value: "${CONTAINER_REGISTRY_CREDENTIALS}"
- name: TEST_JOB_NAME
value: "${TEST_JOB_NAME}"
- name: TEST_JOB_NAMESPACE
value: "${TEST_JOB_NAMESPACE}"
- name: USE_C3IAAS
value: "${USE_C3IAAS}"
- name: ENVIRONMENT
value: "${ENVIRONMENT}"
- name: C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE
value: "${C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAMESPACE}"
- name: C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAME
value: "${C3IAAS_REQUEST_PROJECT_BUILD_CONFIG_NAME}"
- name: C3IAAS_LIFETIME
value: "${C3IAAS_LIFETIME}"
- name: PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE
value: "${PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE}"
- name: MESSAGING_PROVIDER
value: "${MESSAGING_PROVIDER}"
- name: MESSAGING_TOPIC
@@ -139,4 +144,5 @@ objects:
value:
- name: MESSAGE_HEADERS
value:
jenkinsfilePath: openshift/integration/koji/pipelines/templates/mbs-repotracker-trigger.Jenkinsfile
jenkinsfile: |
{% filter indent(width=10) %}{% include "mbs-repotracker-trigger.Jenkinsfile" %}{% endfilter %}

8
openshift/integration/koji/pipelines/templates/snippets/c3i-library-parameters.yaml Normal file
View File

@@ -0,0 +1,8 @@
- name: C3I_LIB_URL
displayName: C3I library git url
required: true
value: "https://pagure.io/c3i-library.git"
- name: C3I_LIB_BRANCH
displayName: C3I library branch
required: true
value: "master"

2
openshift/integration/koji/pipelines/templates/snippets/c3i-library.groovy Normal file
View File

@@ -0,0 +1,2 @@
library identifier: "c3i@${C3I_LIB_BRANCH}", changelog: false,
retriever: modernSCM([$class: 'GitSCMSource', remote: "${C3I_LIB_URL}"])

29
openshift/integration/koji/pipelines/templates/snippets/default-agent.groovy Normal file
View File

@@ -0,0 +1,29 @@
agent {
kubernetes {
cloud "${params.JENKINS_AGENT_CLOUD_NAME}"
label "jenkins-slave-${UUID.randomUUID().toString()}"
serviceAccount "${params.JENKINS_AGENT_SERVICE_ACCOUNT}"
defaultContainer 'jnlp'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
app: "jenkins-${env.JOB_BASE_NAME.take(50).endsWith('-') ? env.JOB_BASE_NAME.take(49): env.JOB_BASE_NAME.take(50)}"
factory2-pipeline-build-number: "${env.BUILD_NUMBER}"
spec:
containers:
- name: jnlp
image: "${params.JENKINS_AGENT_IMAGE}"
imagePullPolicy: Always
tty: true
resources:
requests:
memory: 512Mi
cpu: 300m
limits:
memory: 768Mi
cpu: 500m
"""
}
}

64
openshift/integration/koji/pipelines/templates/snippets/functions.groovy Normal file
View File

@@ -0,0 +1,64 @@
def sendToResultsDB(imageRef, status) {
if (!params.MESSAGING_PROVIDER) {
echo "Message bus is not set. Skipping send of:\nimageRef: ${imageRef}\nstatus: ${status}"
return
}
def (repourl, digest) = imageRef.tokenize('@')
def (registry, reponame) = repourl.split('/', 2)
def image = reponame.split('/').last()
def sendResult = sendCIMessage \
providerName: params.MESSAGING_PROVIDER, \
overrides: [topic: 'VirtualTopic.eng.ci.container-image.test.complete'], \
messageType: 'Custom', \
messageProperties: '', \
messageContent: """
{
"ci": {
"name": "C3I Jenkins",
"team": "DevOps",
"url": "${env.JENKINS_URL}",
"docs": "https://pagure.io/fm-orchestrator/blob/master/f/openshift/integration/koji",
"irc": "#pnt-devops-dev",
"email": "pnt-factory2-devel@redhat.com",
"environment": "${params.ENVIRONMENT}"
},
"run": {
"url": "${env.BUILD_URL}",
"log": "${env.BUILD_URL}/console",
"debug": "",
"rebuild": "${env.BUILD_URL}/rebuild/parametrized"
},
"artifact": {
"type": "container-image",
"repository": "${reponame}",
"digest": "${digest}",
"nvr": "${imageRef}",
"issuer": "c3i-jenkins",
"scratch": ${env.IMAGE_IS_SCRATCH},
"id": "${image}@${digest}"
},
"system":
[{
"os": "${params.JENKINS_AGENT_IMAGE}",
"provider": "openshift",
"architecture": "x86_64"
}],
"type": "integration",
"category": "${params.ENVIRONMENT}",
"status": "${status}",
"xunit": "",
"generated_at": "${new Date().format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone('UTC'))}",
"namespace": "c3i",
"version": "0.1.0"
}
"""
if (sendResult.getMessageId()) {
// echo sent message id and content
echo 'Successfully sent the test result to ResultsDB.'
echo "Message ID: ${sendResult.getMessageId()}"
echo "Message content: ${sendResult.getMessageContent()}"
} else {
echo 'Failed to sent the test result to ResultsDB.'
}
}

14
openshift/integration/koji/pipelines/templates/snippets/get_paas_domain.groovy Normal file
View File

@@ -0,0 +1,14 @@
if (!env.TRIGGER_NAMESPACE) {
env.TRIGGER_NAMESPACE = readFile("/run/secrets/kubernetes.io/serviceaccount/namespace").trim()
}
if(!env.PAAS_DOMAIN) {
openshift.withCluster() {
openshift.withProject(env.TRIGGER_NAMESPACE) {
def testroute = openshift.create('route', 'edge', "test-${env.BUILD_NUMBER}", '--service=test', '--port=8080')
def testhost = testroute.object().spec.host
env.PAAS_DOMAIN = testhost.minus("test-${env.BUILD_NUMBER}-${env.TRIGGER_NAMESPACE}.")
testroute.delete()
}
}
echo "Routes end with ${env.PAAS_DOMAIN}"
}

103
openshift/integration/koji/pipelines/templates/snippets/mbs-integration-test.groovy Normal file
View File

@@ -0,0 +1,103 @@
stage('Run integration tests') {
stages {
stage('Deploy test environment') {
steps {
script {
{% include "snippets/get_paas_domain.groovy" %}
if (!env.PIPELINE_ID) {
env.PIPELINE_ID = "c3i-mbs-${UUID.randomUUID().toString().take(8)}"
}
openshift.withCluster() {
openshift.withProject(params.PIPELINE_AS_A_SERVICE_BUILD_NAMESPACE) {
c3i.buildAndWait(script: this, objs: "bc/pipeline-as-a-service",
'-e', "DEFAULT_IMAGE_TAG=${env.ENVIRONMENT}",
'-e', "PIPELINE_ID=${env.PIPELINE_ID}",
'-e', "WAIVERDB_IMAGE=",
'-e', "C3IAAS_PROJECT=${env.C3IAAS_PROJECT ?: ''}",
'-e', "RESULTSDB_IMAGE=",
'-e', "RESULTSDB_UPDATER_IMAGE=",
'-e', "GREENWAVE_IMAGE=",
'-e', "DATAGREPPER_IMAGE=",
'-e', "DATANOMMER_IMAGE=",
'-e', "MBS_BACKEND_IMAGE=${env.BACKEND_IMAGE_REF}",
'-e', "MBS_FRONTEND_IMAGE=${env.FRONTEND_IMAGE_REF}",
'-e', "PAAS_DOMAIN=${env.PAAS_DOMAIN}"
)
}
}
}
}
}
stage('Run tests') {
steps {
script {
def testcases
if (params.TESTCASES) {
if (params.TESTCASES == 'skip') {
testcases = []
echo 'Skipping integration tests'
} else {
testcases = params.TESTCASES.split()
echo "Using specified list of testcases: ${testcases}"
}
} else {
testcases = findFiles(glob: 'openshift/integration/koji/pipelines/tests/*.groovy').collect {
it.name.minus('.groovy')
}
echo "Using all available testcases: ${testcases}"
}
testcases.each { testcase ->
env.CURRENT_TESTCASE = testcase
echo "Running testcase ${testcase}..."
def test = load "openshift/integration/koji/pipelines/tests/${testcase}.groovy"
test.runTests()
}
}
}
post {
success {
echo "All tests successful"
script {
[env.BACKEND_IMAGE_REF, env.FRONTEND_IMAGE_REF].each {
sendToResultsDB(it, 'passed')
}
}
}
failure {
echo "Testcase ${env.CURRENT_TESTCASE} FAILED"
}
}
}
}
post {
failure {
script {
[env.BACKEND_IMAGE_REF, env.FRONTEND_IMAGE_REF].each {
sendToResultsDB(it, 'failed')
}
openshift.withCluster() {
openshift.withProject(env.PIPELINE_ID) {
echo 'Getting logs from all deployments...'
openshift.selector('pods', ['c3i.redhat.com/pipeline': env.PIPELINE_ID]).logs('--tail 100')
}
}
}
}
cleanup {
script {
if (params.CLEANUP == 'true' && params.USE_C3IAAS != 'true') {
openshift.withCluster() {
openshift.withProject(env.PIPELINE_ID) {
/* Tear down everything we just created */
echo 'Tearing down test resources...'
openshift.selector('all,pvc,configmap,secret',
['c3i.redhat.com/pipeline': env.PIPELINE_ID]).delete('--ignore-not-found=true')
}
}
} else {
echo 'Skipping cleanup'
}
}
}
}
}