C3I: Rewrite test into bash to be re-usable

Move execution logic from groovy scripts to bash. To be possible to run
them from any environment.
Setup pipeline using provion endpoint.

openshift/integration/koji/containers/jenkins-slave/Dockerfile

@@ -31,7 +31,7 @@ ENV HOME=${HOME_DIR} \
 USER root
 
 RUN ${DNF_CMD} install -y \
-    java-1.8.0-openjdk nss_wrapper gettext git \
+    java-1.8.0-openjdk nss_wrapper gettext git jq \
     tar gzip skopeo wget make bind-utils python3-jinja2-cli \
     origin-clients \
     # Jenkins pipeline 'sh' steps seem to require ps

openshift/integration/koji/pipelines/templates/snippets/mbs-integration-test.groovy

@@ -31,27 +31,7 @@ stage('Run integration tests') {
     stage('Run tests') {
       steps {
         script {
-          def testcases
-          if (params.TESTCASES) {
-            if (params.TESTCASES == 'skip') {
-              testcases = []
-              echo 'Skipping integration tests'
-            } else {
-              testcases = params.TESTCASES.split()
-              echo "Using specified list of testcases: ${testcases}"
-            }
-          } else {
-            testcases = findFiles(glob: 'openshift/integration/koji/pipelines/tests/*.groovy').collect {
-              it.name.minus('.groovy')
-            }
-            echo "Using all available testcases: ${testcases}"
-          }
-          testcases.each { testcase ->
-            env.CURRENT_TESTCASE = testcase
-            echo "Running testcase ${testcase}..."
-            def test = load "openshift/integration/koji/pipelines/tests/${testcase}.groovy"
-            test.runTests()
-          }
+          sh "openshift/integration/koji/pipelines/tests/runtests ${env.PIPELINE_ID}"
         }
       }
       post {
@@ -64,7 +44,7 @@ stage('Run integration tests') {
           }
         }
         failure {
-          echo "Testcase ${env.CURRENT_TESTCASE} FAILED"
+          echo "Testcases FAILED"
         }
       }
     }

openshift/integration/koji/pipelines/tests/mbs-cgimport-vars.yaml

@@ -0,0 +1,22 @@
+---
+users:
+  - username: mbs-user-{{ pipeline_id }}
+    name: MBS User {{ pipeline_id }}
+    ldap_groups:
+      - devel
+
+koji:
+  btypes:
+    - module
+  tags:
+    - name: module-f28
+    - name: module-f28-build
+      arches:
+        - x86_64
+      inheritance:
+        - parent: module-f28
+          priority: 0
+  targets:
+    - name: module-f28
+      build_tag: module-f28-build
+      dest_tag: module-f28

openshift/integration/koji/pipelines/tests/module-build-br-virtual-stream

@@ -0,0 +1,71 @@
+#!/bin/bash -ex
+
+# expected environment variables:
+# CACERT = filepath with CA cert
+# MBS_FRONTEND_HOST
+# KOJI_CONFIG = path to koji configuration
+
+# Kerberos ticket with user having permissions to build module
+
+if [ -z "$CACERT" ] || [ -z "$MBS_FRONTEND_HOST" ] || [ -z "$KOJI_CONFIG" ] || ! klist &>/dev/null ; then
+    echo "Missing environment configuration"
+    exit 1
+fi
+
+RESPONSE=$(mktemp)
+HTTP_CODE=$(mktemp)
+MODULE_REQUEST=$(mktemp)
+
+cat > $MODULE_REQUEST << EOF
+{ "modulemd": "$(sed 's/$/\\n/' ${TEST_DIR}/modulemd.yaml | tr -d '\n')"}
+EOF
+
+curl -s --negotiate -u : \
+     --cacert $CACERT \
+     -H 'Content-Type: application/json' \
+     -H 'Accept: application/json' \
+     -d @$MODULE_REQUEST \
+     -o $RESPONSE \
+     -w '%{http_code}' \
+     https://${MBS_FRONTEND_HOST}/module-build-service/1/module-builds/ > $HTTP_CODE
+
+if [ "$(cat $HTTP_CODE)" != "201" ]; then
+    echo "HTTP code was $(cat $HTTP_CODE), not 201"
+    exit 1
+fi
+
+MODULE_ID=$(jq '.id' $RESPONSE)
+i=0
+while true; do
+    sleep 5
+    i=$((i + 1))
+    if [ $i -gt 100 ]; then
+        echo "Module build has timed out"
+        exit 1
+    fi
+    curl -s --cacert $CACERT \
+         -o $RESPONSE \
+         -w '%{stderr}%{http_code}' \
+         https://${MBS_FRONTEND_HOST}/module-build-service/1/module-builds/$MODULE_ID 2> $HTTP_CODE
+    if [ "$(cat $HTTP_CODE)" != "200" ]; then
+        echo "HTTP code was $(cat $HTTP_CODE), not 200"
+        exit 1
+    fi
+    STATE=$(jq -r '.state_name' $RESPONSE)
+    if [ "$STATE" == "failed" ]; then
+        echo "Module build failed"
+        exit 1
+    elif [ "$STATE" == "ready" ]; then
+        echo "Module build is ready"
+    else
+        continue
+    fi
+    BR_PLATFORM_STREAM=$(jq -r '.buildrequires.platform.stream' $RESPONSE)
+    if [ "${BR_PLATFORM_STREAM}" != "f28" ]; then
+        echo "Module $MODULE_ID buildrequires platform:${BR_PLATFORM_STREAM}, \
+          but it should buildrequire platform:f28"
+        exit 1
+    fi
+    echo "All tests passed"
+    break
+done

openshift/integration/koji/pipelines/tests/module-build-br-virtual-stream.groovy

@@ -1,106 +0,0 @@
-// Build an empty module that buildrequires a virtual stream
-import groovy.json.JsonOutput
-
-def runTests() {
-  def koji_admin = controller.getVar('KOJI_ADMIN')
-  def clientcert = controller.httpGet("/ca/${koji_admin}", true)
-  def koji_ssl_host = controller.getVar('KOJI_HUB_HOST')
-  def mbs_host = controller.getVar('MBS_FRONTEND_HOST')
-  def ca_cert = controller.httpGet("/ca/cacert")
-  koji.setConfig("https://${koji_ssl_host}/kojihub",
-                 "https://${koji_ssl_host}/kojifiles",
-                 clientcert.cert, clientcert.key, ca_cert)
-  def tags = koji.callMethod("listTags")
-  if (!tags.any { it.name == "module-f28" }) {
-    koji.addTag("module-f28")
-  }
-  if (!tags.any { it.name == "module-f28-build" }) {
-    koji.addTag("module-f28-build", "--parent=module-f28", "--arches=x86_64")
-  }
-  try {
-    // There's currently no way to query whether a given user has CG access, so just add it
-    // and hope no one else has already done it.
-    koji.runCmd("grant-cg-access", koji_admin, "module-build-service", "--new")
-  } catch (ex) {
-    echo "Granting cg-access to ${koji_admin} failed, assuming it was already provided in a previous test"
-  }
-
-  if (!koji.callMethod("listBTypes").any { it.name == "module" }) {
-    koji.callMethodLogin("addBType", "module")
-  }
-
-  def testmodule = """
-  document: modulemd
-  version: 1
-  data:
-    summary: A test module in all its beautiful beauty
-    description: This module buildrequires a virtual stream of the platform module
-    name: testmodule
-    stream: buildrequire_virtual_stream
-    license:
-      module: [ MIT ]
-    dependencies:
-      buildrequires:
-          platform: fedora
-      requires:
-          platform: fedora
-    references:
-      community: https://docs.pagure.org/modularity/
-      documentation: https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules
-  """
-
-  writeFile file: 'ca-cert.pem', text: ca_cert
-  def url = "https://${mbs_host}/module-build-service/1/module-builds/"
-  def curlargs = """
-    --cacert ca-cert.pem \
-    -H 'Content-Type: application/json' \
-    -H 'Accept: application/json' \
-    -d @buildparams.json \
-    -o response.json \
-    -w '%{http_code}'
-  """.trim()
-  def http_code, response
-  if (controller.getVar("KRB5_REALM")) {
-    writeFile file: 'buildparams.json', text: JsonOutput.toJson([modulemd: testmodule])
-    krb5.withKrb(controller.getKrb5Vars(koji_admin)) {
-      http_code = sh script: "curl --negotiate -u : $curlargs $url", returnStdout: true
-      response = readFile file: 'response.json'
-    }
-  } else {
-    writeFile file: 'buildparams.json', text: JsonOutput.toJson([modulemd: testmodule, owner: env.KOJI_ADMIN])
-    http_code = sh script: "curl $curlargs $url", returnStdout: true
-    response = readFile file: 'response.json'
-  }
-  if (http_code != '201') {
-    echo "Response code was ${http_code}, output was ${response}"
-    error "POST response code was ${http_code}, not 201"
-  }
-  def buildinfo = readJSON(text: response)
-  timeout(10) {
-    waitUntil {
-      def resp = httpRequest url: "${url}${buildinfo.id}", ignoreSslErrors: true
-      if (resp.status != 200) {
-        echo "Response code was ${resp.status}, output was ${resp.content}"
-        error "GET response code was ${resp.status}, not 200"
-      }
-      def modinfo = readJSON(text: resp.content)
-      if (modinfo.state_name == "failed") {
-        error "Module ${modinfo.id} (${modinfo.name}) is in the ${modinfo.state_name} state"
-      } else if (modinfo.state_name != "ready") {
-        echo "Module ${modinfo.id} (${modinfo.name}) is in the ${modinfo.state_name} state, not ready"
-        return false
-      }
-      def br_platform_stream = modinfo.buildrequires.platform.stream
-      if (br_platform_stream != "f28") {
-        echo "Module ${modinfo.id} (${modinfo.name}) buildrequires platform:${br_platform_stream}, \
-          but it should buildrequire platform:f28"
-        return false
-      }
-
-      echo "All checks passed"
-      return true
-    }
-  }
-}
-
-return this;

openshift/integration/koji/pipelines/tests/module-build-cgimport

@@ -0,0 +1,81 @@
+#!/bin/bash -ex
+
+# expected environment variables:
+# CACERT = filepath with CA cert
+# MBS_FRONTEND_HOST
+# KOJI_CONFIG = path to koji configuration
+
+# Kerberos ticket with user having permissions to build module
+
+if [ -z "$CACERT" ] || [ -z "$MBS_FRONTEND_HOST" ] || [ -z "$KOJI_CONFIG" ] || ! klist &>/dev/null ; then
+    echo "Missing environment configuration"
+    exit 1
+fi
+
+RESPONSE=$(mktemp)
+HTTP_CODE=$(mktemp)
+SCMURL="https://src.fedoraproject.org/forks/mikeb/modules/testmodule.git?#8b3fb16160f899ce10905faf570f110d52b91154"
+EXISTING_MODULE=$(curl -s --cacert $CACERT https://${MBS_FRONTEND_HOST}/module-build-service/1/module-builds/ | jq ".items[] | select(.scmurl == \"${SCMURL}\" and .state != 4) | .id")
+if [ -n "$EXISTING_MODULE" ]; then
+    echo "Marking existing module $EXISTING_MODULE as failed so it can be rebuilt..."
+    curl -s --negotiate -u : \
+         --cacert $CACERT \
+         -X PATCH \
+         -d '{"state": "failed"}' \
+         https://${MBS_FRONTEND_HOST}/module-build-service/1/module-builds/${EXISTING_MODULE} | jq
+fi
+curl -s --negotiate -u : \
+     --cacert $CACERT \
+     -H 'Content-Type: application/json' \
+     -H 'Accept: application/json' \
+     -d "{\"scmurl\": \"${SCMURL}\", \"branch\": \"empty-f28\"}" \
+     -o $RESPONSE \
+     -w '%{stderr}%{http_code}' \
+     https://${MBS_FRONTEND_HOST}/module-build-service/1/module-builds/ 2> $HTTP_CODE
+if [ "$(cat $HTTP_CODE)" != "201" ]; then
+    echo "HTTP code was $(cat $HTTP_CODE), not 201"
+    exit 1
+fi
+MODULE_ID=$(jq '.id' $RESPONSE)
+i=0
+while true; do
+    sleep 5
+    i=$((i + 1))
+    if [ $i -gt 100 ]; then
+        echo "Module build has timed out"
+        exit 1
+    fi
+    curl -s --cacert $CACERT \
+         -o $RESPONSE \
+         -w '%{stderr}%{http_code}' \
+         https://${MBS_FRONTEND_HOST}/module-build-service/1/module-builds/$MODULE_ID 2> $HTTP_CODE
+    if [ "$(cat $HTTP_CODE)" != "200" ]; then
+        echo "HTTP code was $(cat $HTTP_CODE), not 200"
+        exit 1
+    fi
+    STATE=$(jq -r '.state_name' $RESPONSE)
+    if [ "$STATE" == "failed" ]; then
+        echo "Module build failed"
+        exit 1
+    elif [ "$STATE" == "ready" ]; then
+        echo "Module build is ready"
+    else
+        continue
+    fi
+    TESTMODULE="$(koji -c $KOJI_CONFIG -q list-builds --package testmodule)"
+    if [ -z "$TESTMODULE" ]; then
+        echo "No builds of testmodule"
+        exit 1
+    else
+        echo $TESTMODULE
+    fi
+    TESTMODULEDEVEL="$(koji -c $KOJI_CONFIG -q list-builds --package testmodule-devel)"
+    if [ -z "$TESTMODULEDEVEL" ]; then
+        echo "No builds of testmodule-devel"
+        exit 1
+    else
+        echo $TESTMODULEDEVEL
+    fi
+    echo "All tests passed"
+    break
+done

openshift/integration/koji/pipelines/tests/module-build-cgimport.groovy

@@ -1,97 +0,0 @@
-// Build an empty module and verify that the CGImport works correctly
-
-def runTests() {
-  def koji_admin = controller.getVar('KOJI_ADMIN')
-  def clientcert = controller.httpGet("/ca/${koji_admin}", true)
-  def koji_ssl_host = controller.getVar('KOJI_HUB_HOST')
-  def mbs_host = controller.getVar('MBS_FRONTEND_HOST')
-  def ca_cert = controller.httpGet("/ca/cacert")
-  koji.setConfig("https://${koji_ssl_host}/kojihub",
-                 "https://${koji_ssl_host}/kojifiles",
-                 clientcert.cert, clientcert.key, ca_cert)
-  def tags = koji.callMethod("listTags")
-  if (!tags.any { it.name == "module-f28" }) {
-    koji.addTag("module-f28")
-  }
-  if (!tags.any { it.name == "module-f28-build" }) {
-    koji.addTag("module-f28-build", "--parent=module-f28", "--arches=x86_64")
-  }
-  try {
-    // There's currently no way to query whether a given user has CG access, so just add it
-    // and hope no one else has already done it.
-    koji.runCmd("grant-cg-access", koji_admin, "module-build-service", "--new")
-  } catch (ex) {
-    echo "Granting cg-access to ${koji_admin} failed, assuming it was already provided in a previous test"
-  }
-
-  if (!koji.callMethod("listBTypes").any { it.name == "module" }) {
-    koji.callMethodLogin("addBType", "module")
-  }
-
-  writeFile file: 'ca-cert.pem', text: ca_cert
-  def url = "https://${mbs_host}/module-build-service/1/module-builds/"
-  def curlargs = """
-    --cacert ca-cert.pem \
-    -H 'Content-Type: application/json' \
-    -H 'Accept: application/json' \
-    -d @buildparams.json \
-    -o response.json \
-    -w '%{http_code}'
-  """.trim()
-  def http_code, response
-  if (controller.getVar("KRB5_REALM")) {
-    writeFile file: 'buildparams.json', text: """
-      {"scmurl": "https://src.fedoraproject.org/forks/mikeb/modules/testmodule.git?#8b3fb16160f899ce10905faf570f110d52b91154",
-       "branch": "empty-f28"}
-    """
-    krb5.withKrb(controller.getKrb5Vars(koji_admin)) {
-      http_code = sh script: "curl --negotiate -u : $curlargs $url", returnStdout: true
-      response = readFile file: 'response.json'
-    }
-  } else {
-    writeFile file: 'buildparams.json', text: """
-      {"scmurl": "https://src.fedoraproject.org/forks/mikeb/modules/testmodule.git?#8b3fb16160f899ce10905faf570f110d52b91154",
-       "branch": "empty-f28",
-       "owner":  "${koji_admin}"}
-    """
-    http_code = sh script: "curl $curlargs $url", returnStdout: true
-    response = readFile file: 'response.json'
-  }
-  if (http_code != '201') {
-    echo "Response code was ${http_code}, output was ${response}"
-    error "POST response code was ${http_code}, not 201"
-  }
-  def buildinfo = readJSON(text: response)
-  timeout(10) {
-    waitUntil {
-      def resp = httpRequest url: "${url}${buildinfo.id}", ignoreSslErrors: true
-      if (resp.status != 200) {
-        echo "Response code was ${resp.status}, output was ${resp.content}"
-        error "GET response code was ${resp.status}, not 200"
-      }
-      def modinfo = readJSON(text: resp.content)
-      if (modinfo.state_name == "failed") {
-        error "Module ${modinfo.id} (${modinfo.name}) is in the ${modinfo.state_name} state"
-      } else if (modinfo.state_name != "ready") {
-        echo "Module ${modinfo.id} (${modinfo.name}) is in the ${modinfo.state_name} state, not ready"
-        return false
-      }
-      def builds = koji.listBuilds()
-      echo "Builds: ${builds}"
-      def build = builds.find { it.name == "testmodule" }
-      if (!build) {
-        echo "Could not find a build of testmodule"
-        return false
-      }
-      def develbuild = builds.find { it.name == "testmodule-devel" }
-      if (!develbuild) {
-        echo "Could not find a build of testmodule-devel"
-        return false
-      }
-      echo "All checks passed"
-      return true
-    }
-  }
-}
-
-return this;

openshift/integration/koji/pipelines/tests/module-build-init

@@ -0,0 +1,125 @@
+#!/bin/bash -ex
+
+# expected environment variables:
+# CACERT = filepath with CA cert
+# MBS_FRONTEND_HOST
+# KOJI_CONFIG = path to koji configuration
+
+# Kerberos ticket with user having permissions to build module
+
+if [ -z "$CACERT" ] || [ -z "$MBS_FRONTEND_HOST" ] || [ -z "$KOJI_CONFIG" ] || ! klist &>/dev/null ; then
+    echo "Missing environment configuration"
+    exit 1
+fi
+
+RESPONSE=$(mktemp)
+HTTP_CODE=$(mktemp)
+MODULE_REQUEST=$(mktemp)
+
+cat > $MODULE_REQUEST << EOF
+{"scmurl": "https://src.fedoraproject.org/modules/testmodule.git?#9c589780e1dd1698dc64dfa28d30014ad18cad32", "branch": "f28"}
+EOF
+
+curl -s --negotiate -u : \
+     --cacert $CACERT \
+     -H 'Content-Type: application/json' \
+     -H 'Accept: application/json' \
+     -d @$MODULE_REQUEST \
+     -o $RESPONSE \
+     -w '%{http_code}' \
+     https://${MBS_FRONTEND_HOST}/module-build-service/1/module-builds/ > $HTTP_CODE
+
+if [ "$(cat $HTTP_CODE)" != "201" ]; then
+    echo "HTTP code was $(cat $HTTP_CODE), not 201"
+    exit 1
+fi
+
+MODULE_ID=$(jq '.id' $RESPONSE)
+TEST_TARGET_TAG_INFO_FILE=$(mktemp)
+i=0
+while true; do
+    sleep 5
+    i=$((i + 1))
+    if [ $i -gt 100 ]; then
+        echo "Module build has timed out"
+        exit 1
+    fi
+    curl -s --cacert $CACERT \
+         -o $RESPONSE \
+         -w '%{stderr}%{http_code}' \
+         https://${MBS_FRONTEND_HOST}/module-build-service/1/module-builds/$MODULE_ID 2> $HTTP_CODE
+    if [ "$(cat $HTTP_CODE)" != "200" ]; then
+        echo "HTTP code was $(cat $HTTP_CODE), not 200"
+        exit 1
+    fi
+    STATE=$(jq -r '.state_name' $RESPONSE)
+    if [ "$STATE" == "failed" ]; then
+        echo "Module build failed"
+        exit 1
+    elif [ "$STATE" != "build" ]; then
+        echo "Module ${MODULE_ID} is in the $STATE state, not build"
+        continue
+    fi
+    TEST_TARGET="$(koji -c $KOJI_CONFIG -q list-targets | awk '$1~"^module-testmodule-" {print $1, $2}')"
+    if [ -z "$TEST_TARGET" ]; then
+        echo "Could not find module target"
+        continue
+    fi
+    TEST_TARGET_NAME=$(echo ${TEST_TARGET} | awk '{print $1}')
+    TEST_TARGET_BUILD_TAG=$(echo ${TEST_TARGET} | awk '{print $2}')
+    echo "Target: ${TEST_TARGET_NAME}"
+    koji -c $KOJI_CONFIG taginfo ${TEST_TARGET_BUILD_TAG} > ${TEST_TARGET_TAG_INFO_FILE}
+
+    if ! grep -q "Arches: x86_64" ${TEST_TARGET_TAG_INFO_FILE}; then
+       echo "${TEST_TARGET_BUILD_TAG} does not have arches set to x86_64"
+       continue
+    fi
+
+    if ! grep -q "Required permission: 'admin'" ${TEST_TARGET_TAG_INFO_FILE}; then
+       echo "${TEST_TARGET_BUILD_TAG} does not have perm set to admin"
+       continue
+    fi
+    if ! grep -q "  mock.package_manager : 'dnf'" ${TEST_TARGET_TAG_INFO_FILE}; then
+       echo "${TEST_TARGET_BUILD_TAG} is not configured to use dnf"
+       continue
+    fi
+    if ! grep -q "  repo_include_all : True" ${TEST_TARGET_TAG_INFO_FILE}; then
+       echo "${TEST_TARGET_BUILD_TAG} is not configured with repo_include_all"
+       continue
+    fi
+    if ! koji -c $KOJI_CONFIG list-tag-inheritance ${TEST_TARGET_BUILD_TAG} | grep -q 'module-f28-build'; then
+       echo "module-f28-build not in inheritance of  ${TEST_TARGET_BUILD_TAG}"
+       continue
+    fi
+    if [ "$(koji -c $KOJI_CONFIG list-groups ${TEST_TARGET_BUILD_TAG} | grep 'bash:\|rpm-build:\|module-build-macros:' | wc -l)" != 6 ]; then
+       echo "${TEST_TARGET_BUILD_TAG} does not have required packages in the srpm-build or build group "
+       continue
+    fi
+    BUILD_TASK=$(koji -c $KOJI_CONFIG list-tasks | grep 'build (')
+    if [ -z "${BUILD_TASK}" ]; then
+       echo "No build task has been created"
+       continue
+    fi
+    if ! echo ${BUILD_TASK} | grep -q "module-build-macros"; then
+        echo "The build task is not building module-build-macros"
+        continue
+    fi
+    if ! echo ${BUILD_TASK} | grep -q "\.src\.rpm"; then
+        echo "The build task is not building from an SRPM"
+        continue
+    fi
+
+    NEW_REPO_TASK=$(koji -c $KOJI_CONFIG list-tasks | grep 'newRepo (')
+    if [ -z "${NEW_REPO_TASK}" ]; then
+       echo "No newRepo task has been created"
+       continue
+    fi
+
+    if ! echo $NEW_REPO_TASK | grep -q ${TEST_TARGET_BUILD_TAG}; then
+       echo "The newRepo task is not associated with the correct tag"
+       continue
+    fi
+
+    echo "All tests passed"
+    break
+done

openshift/integration/koji/pipelines/tests/modulemd.yaml

@@ -0,0 +1,17 @@
+document: modulemd
+version: 1
+data:
+  summary: A test module in all its beautiful beauty
+  description: This module buildrequires a virtual stream of the platform module
+  name: testmodule
+  stream: buildrequire_virtual_stream
+  license:
+    module: [ MIT ]
+  dependencies:
+    buildrequires:
+        platform: fedora
+    requires:
+        platform: fedora
+  references:
+    community: https://docs.pagure.org/modularity/
+    documentation: https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules

openshift/integration/koji/pipelines/tests/runtests

@@ -0,0 +1,54 @@
+#!/bin/bash -ex
+
+export PIPELINE_ID=$1
+if [ -z "$PIPELINE_ID" ]; then
+    echo You must specify the pipeline ID
+    exit 1
+fi
+
+if [ "$TESTCASES" == "skip" ]; then
+    echo "TESTCASES=skip defined, skipping tests"
+    exit 0
+fi
+
+export TEST_DIR=$(realpath $(dirname $0))
+CONTROLLER=http://${PIPELINE_ID}.cloud.paas.psi.redhat.com
+echo "Provisioning pipeline services..."
+curl -X POST -F data=@${TEST_DIR}/mbs-cgimport-vars.yaml $CONTROLLER/scripts/provision
+CERT_DIR=$(mktemp -d)
+export CACERT="${CERT_DIR}/ca.crt"
+curl -s ${CONTROLLER}/ca/cacert > $CACERT
+export KRB5_CONFIG=$(mktemp)
+curl -so $KRB5_CONFIG ${CONTROLLER}/krb5/configfile
+export MBS_FRONTEND_HOST="$(curl -s ${CONTROLLER}/vars/MBS_FRONTEND_HOST)"
+# The MBS user is defined in the Ansible vars file
+export MBS_USER="mbs-user-${PIPELINE_ID}"
+export MBS_USER_PASSWORD=$(curl -s ${CONTROLLER}/krb5/principal/${MBS_USER})
+kinit -V $MBS_USER <<<$MBS_USER_PASSWORD
+
+CLIENT_CERTS="${CERT_DIR}/client.pem"
+curl -s ${CONTROLLER}/ca/${MBS_USER}/cert > ${CLIENT_CERTS}
+curl -s ${CONTROLLER}/ca/${MBS_USER}/key >> ${CLIENT_CERTS}
+
+KOJI_HUB_HOST="$(curl -s ${CONTROLLER}/vars/KOJI_HUB_HOST)"
+export KOJI_CONFIG=$(mktemp)
+cat > $KOJI_CONFIG <<EOF
+[koji]
+server = https://${KOJI_HUB_HOST}/kojihub
+weburl = https://${KOJI_HUB_HOST}/koji
+serverca = $CACERT
+authtype = ssl
+cert = ${CLIENT_CERTS}
+EOF
+
+for TEST_FILE in $(ls ${TEST_DIR}); do
+    TEST="${TEST_DIR}/${TEST_FILE}"
+    if [ -x "$TEST" ] && [ "${TEST_FILE}" != "runtests" ]; then
+       if [ -z "$TESTCASES" ] || echo "$TESTCASES" | grep "${TEST_FILE}"; then
+           echo "Running test ${TEST_FILE}"
+           $TEST
+       else
+           echo "Skipping test ${TEST_FILE}"
+       fi
+    fi
+done