mirror of
https://pagure.io/fm-orchestrator.git
synced 2026-02-13 01:54:59 +08:00
a5f6d8f136
Until now, it was assumed that the module-build command returned only one build, so it was only one build_id. However, it is possible that the module-build command will build more than one builds and therefore a list of build_ids is needed. Also is needed to watch and cancel more than one build. For this reason run, watch, and cancel methods are methods of the PackagingUtility class instead of Build class. Run method returns list of Build objects instead of build_id. And it's also possible to cancel and to watch on all generated module builds.
392 lines
13 KiB
Python
392 lines
13 KiB
Python
# -*- coding: utf-8 -*-
|
|
# SPDX-License-Identifier: MIT
|
|
|
|
import re
|
|
import sys
|
|
import time
|
|
|
|
from kobo import rpmlib
|
|
import koji
|
|
import yaml
|
|
import requests
|
|
import tempfile
|
|
import sh
|
|
|
|
our_sh = sh(_out=sys.stdout, _err=sys.stderr, _tee=True)
|
|
from our_sh import Command, git, pushd # noqa
|
|
|
|
|
|
class Koji:
|
|
"""Wrapper class to work with Koji
|
|
|
|
:attribute string _server: URL of the Koji hub
|
|
:attribute string _topurl: URL of the top-level Koji download location
|
|
:attribute koji.ClientSession _session: Koji session
|
|
:attribute koji.PathInfo _pathinfo: Koji path
|
|
"""
|
|
|
|
def __init__(self, server, topurl):
|
|
self._server = server
|
|
self._topurl = topurl
|
|
self._session = koji.ClientSession(self._server)
|
|
self._pathinfo = koji.PathInfo(self._topurl)
|
|
|
|
def get_build(self, nvr_dict):
|
|
"""Koji build data for NVR
|
|
|
|
:param dict nvr_dict: NVR dictionary as expected by kobo.rpmlib.make_nvr()
|
|
:return: Dictionary with Koji build data or None, if build is not found
|
|
:rtype: dict or None
|
|
"""
|
|
nvr_string = rpmlib.make_nvr(nvr_dict)
|
|
return self._session.getBuild(nvr_string)
|
|
|
|
def get_modulemd(self, cg_build):
|
|
"""Modulemd file of the build from koji archive
|
|
|
|
:param cg_build: koji build object
|
|
:return: Modulemd file
|
|
:rtype: dict
|
|
"""
|
|
url = self._pathinfo.typedir(cg_build, 'module')
|
|
r = requests.get(f"{url}/modulemd.txt")
|
|
r.raise_for_status()
|
|
return yaml.safe_load(r.content)
|
|
|
|
|
|
class Repo:
|
|
"""Wrapper class to work with module git repositories
|
|
|
|
:attribute string module_name: name of the module stored in this repo
|
|
:attribute dict _modulemd: Modulemd file as read from the repo
|
|
"""
|
|
|
|
def __init__(self, module_name):
|
|
self.module_name = module_name
|
|
self._modulemd = None
|
|
self._version = None
|
|
|
|
@property
|
|
def modulemd(self):
|
|
"""Modulemd file as read from the repo
|
|
|
|
:return: Modulemd file as read from the repo
|
|
:rtype: dict
|
|
"""
|
|
if self._modulemd is None:
|
|
modulemd_file = self.module_name + ".yaml"
|
|
with open(modulemd_file, "r") as f:
|
|
self._modulemd = yaml.safe_load(f)
|
|
return self._modulemd
|
|
|
|
@property
|
|
def components(self):
|
|
"""List of components as defined in the modulemd file
|
|
|
|
:return: List of components as defined in the modulemd file
|
|
:rtype: list of strings
|
|
"""
|
|
return list(self.modulemd["data"]["components"]["rpms"])
|
|
|
|
@property
|
|
def platform(self):
|
|
"""
|
|
List of platforms in the modulemd file, obtaining values differs on version
|
|
|
|
:return: List of platforms in the modulemd file
|
|
:rtype: list of strings
|
|
"""
|
|
if self._version is None:
|
|
self._version = self._modulemd["version"]
|
|
if self._version == 1:
|
|
return [self._modulemd["data"]["dependencies"]["buildrequires"].get("platform")]
|
|
elif self._version == 2:
|
|
return self._modulemd["data"]["dependencies"][0]["buildrequires"].get("platform")
|
|
|
|
def bump(self):
|
|
"""Create a "bump" commit"""
|
|
args = [
|
|
"--allow-empty",
|
|
"-m",
|
|
"Bump"
|
|
]
|
|
git("commit", *args)
|
|
git("push")
|
|
|
|
|
|
class PackagingUtility:
|
|
"""Wrapper class to work with the packaging utility configured for the tests
|
|
|
|
:attribute sh.Command _packaging_utility: packaging utility command used to
|
|
kick off this build
|
|
:attribute string _mbs_api: URL of the MBS API (including trailing '/')
|
|
"""
|
|
|
|
def __init__(self, packaging_utility, mbs_api):
|
|
self._packaging_utility = Command(packaging_utility).bake(
|
|
_out=sys.stdout, _err=sys.stderr, _tee=True
|
|
)
|
|
self._mbs_api = mbs_api
|
|
|
|
def run(self, *args, reuse=None):
|
|
"""Run one or more module builds
|
|
|
|
:param args: Options and arguments for the build command
|
|
:param int reuse: An optional MBS build id or a list of MBS build
|
|
ids to be reused for this run.
|
|
When specified, the corresponding module build(s) will be used,
|
|
instead of triggering and waiting for new one(s) to finish.
|
|
Intended to be used while developing the tests.
|
|
:return: list of Build objects for the MBS builds created
|
|
:rtype: list of Build objects
|
|
"""
|
|
build_ids = []
|
|
|
|
if reuse is not None:
|
|
if isinstance(reuse, list):
|
|
build_ids = reuse
|
|
else:
|
|
build_ids = [reuse]
|
|
else:
|
|
stdout = self._packaging_utility("module-build", *args).stdout.decode("utf-8")
|
|
build_ids = re.findall(self._mbs_api + r"module-builds/(\d+)", stdout)
|
|
return [Build(self._mbs_api, int(build_id)) for build_id in build_ids]
|
|
|
|
def watch(self, builds):
|
|
"""Watch one or more builds till the finish
|
|
|
|
:param list builds: list of Build objects of the builds to be watched.
|
|
:return: Stdout of the watch command
|
|
:rtype: string
|
|
"""
|
|
stdout = self._packaging_utility(
|
|
"module-build-watch", [str(build.id) for build in builds]
|
|
).stdout.decode("utf-8")
|
|
|
|
return stdout
|
|
|
|
def cancel(self, build):
|
|
"""Cancel the module build
|
|
|
|
:param list build: the Build object of the module build to be cancelled.
|
|
:return: Standard output of the "module-build-cancel <build id=""> command
|
|
:rtype: str
|
|
"""
|
|
stdout = self._packaging_utility("module-build-cancel", build.id).stdout.decode(
|
|
"utf-8")
|
|
return stdout
|
|
|
|
|
|
class Build:
|
|
"""Wrapper class to work with module builds
|
|
|
|
:attribute string _mbs_api: URL of the MBS API (including trailing '/')
|
|
:attribute int _build_id: id of this MBS module build
|
|
:attribute string _data: Module build data cache for this build fetched from MBS
|
|
:attribute string _module_build_data: Verbose module build data cache for this build
|
|
"""
|
|
|
|
def __init__(self, mbs_api, build_id):
|
|
self._mbs_api = mbs_api
|
|
self._data = None
|
|
self._component_data = None
|
|
self._build_id = build_id
|
|
self._module_build_data = None
|
|
|
|
@property
|
|
def id(self):
|
|
return self._build_id
|
|
|
|
@property
|
|
def data(self):
|
|
"""Module build data cache for this build fetched from MBS"""
|
|
if self._data is None and self._build_id:
|
|
r = requests.get(f"{self._mbs_api}module-builds/{self._build_id}")
|
|
r.raise_for_status()
|
|
self._data = r.json()
|
|
return self._data
|
|
|
|
@property
|
|
def component_data(self):
|
|
"""Component data for the module build"""
|
|
if self._component_data is None and self._build_id:
|
|
params = {
|
|
"module_build": self._build_id,
|
|
"verbose": True,
|
|
}
|
|
r = requests.get(f"{self._mbs_api}component-builds/", params=params)
|
|
r.raise_for_status()
|
|
self._component_data = r.json()
|
|
return self._component_data
|
|
|
|
@property
|
|
def module_build_data(self):
|
|
"""Verbose module build
|
|
|
|
:return: Dictionary of the verbose module build parameters
|
|
:rtype: dict
|
|
"""
|
|
if self._build_id:
|
|
params = {
|
|
"verbose": True,
|
|
}
|
|
r = requests.get(f"{self._mbs_api}module-builds/{self._build_id}", params=params)
|
|
r.raise_for_status()
|
|
self._module_build_data = r.json()
|
|
return self._module_build_data
|
|
|
|
@property
|
|
def state_name(self):
|
|
"""Name of the state of this module build"""
|
|
return self.data["state_name"]
|
|
|
|
def components(self, state=None, batch=None, package=None):
|
|
"""Components of this module build, optionally filtered based on properties
|
|
|
|
:param string state: Koji build state the components should be in
|
|
:param int batch: the number of the batch the components should be in
|
|
:param string package: name of the component (package)
|
|
:return: List of filtered components
|
|
:rtype: list of dict
|
|
"""
|
|
filtered = self.component_data["items"]
|
|
if batch is not None:
|
|
filtered = filter(lambda x: x["batch"] == batch, filtered)
|
|
if state is not None:
|
|
filtered = filter(lambda x: x["state_name"] == state, filtered)
|
|
if package is not None:
|
|
filtered = filter(lambda x: x["package"] == package, filtered)
|
|
|
|
return list(filtered)
|
|
|
|
def component_names(self, state=None, batch=None, package=None):
|
|
"""Component names of this module build, optionally filtered based on properties
|
|
|
|
:param string state: Koji build state the components should be in
|
|
:param int batch: the number of the batch the components should be in
|
|
:param string package: name of component (package):
|
|
:return: List of components packages
|
|
:rtype: list of strings
|
|
"""
|
|
components = self.components(state, batch, package)
|
|
return [item["package"] for item in components]
|
|
|
|
def component_task_ids(self):
|
|
"""Dictionary containing all names of packages from build and appropriate task ids
|
|
|
|
:return: Dictionary containing name of packages and their task id
|
|
:rtype: dict
|
|
"""
|
|
return {comp["package"]: comp["task_id"] for comp in self.components()}
|
|
|
|
def batches(self):
|
|
"""
|
|
Components of the module build separated in sets according to batches
|
|
|
|
:return: list of components according to batches
|
|
:rtype: list of sets
|
|
"""
|
|
comps_data = sorted(self.component_data["items"], key=lambda x: x["batch"])
|
|
batch_count = comps_data[-1]["batch"]
|
|
batches = batch_count * [set()]
|
|
for data in comps_data:
|
|
batch = data["batch"]
|
|
package = data["package"]
|
|
batches[batch - 1] = batches[batch - 1].union({package})
|
|
|
|
return batches
|
|
|
|
def wait_for_koji_task_id(self, package, batch, timeout=300, sleep=10):
|
|
"""Wait until the component is submitted to Koji (has a task_id)
|
|
|
|
:param string package: name of component (package)
|
|
:param int batch: the number of the batch the components should be in
|
|
:param int timeout: time in seconds
|
|
:param int sleep: time in seconds
|
|
"""
|
|
start = time.time()
|
|
while time.time() - start <= timeout:
|
|
# Clear cached data
|
|
self._component_data = None
|
|
components = self.components(package=package, batch=batch)
|
|
# Wait until the right component appears and has a task_id
|
|
if components and components[0]["task_id"]:
|
|
return components[0]["task_id"]
|
|
time.sleep(sleep)
|
|
|
|
raise RuntimeError(
|
|
f'Koji task for "{package}" did not start in {timeout} seconds'
|
|
)
|
|
|
|
def nvr(self, name_suffix=""):
|
|
"""NVR dictionary of this module build
|
|
|
|
:param string name_suffix: an optional suffix for the name component of the NVR
|
|
:return: dictionary with NVR components
|
|
:rtype: dict
|
|
"""
|
|
return {
|
|
"name": f'{self.data["name"]}{name_suffix}',
|
|
"version": self.data["stream"].replace("-", "_"),
|
|
"release": f'{self.data["version"]}.{self.data["context"]}',
|
|
}
|
|
|
|
def was_cancelled(self):
|
|
"""Checking in the status trace if module was canceled
|
|
|
|
:return: Whether exists required status
|
|
:rtype: bool
|
|
"""
|
|
for item in self.module_build_data["state_trace"]:
|
|
if (
|
|
item["reason"] is not None
|
|
and "Canceled" in item["reason"]
|
|
and item["state_name"] == "failed"
|
|
):
|
|
return True
|
|
return False
|
|
|
|
|
|
class Component:
|
|
"""Wrapper class to work with git repositories of components
|
|
|
|
:attribute string module_name: name of the module stored in this repo
|
|
:attribute string branch: branch of the git repo that will be used
|
|
:attribute TemporaryDirectory _clone_dir: directory where is the clone of the repo
|
|
"""
|
|
def __init__(self, module_name, branch):
|
|
self._module_name = module_name
|
|
self._branch = branch
|
|
self._clone_dir = None
|
|
|
|
def __del__(self):
|
|
self._clone_dir.cleanup()
|
|
|
|
def clone(self, packaging_utility):
|
|
"""Clone the git repo of the component to be used by the test in a temporary location
|
|
|
|
Directory of the clone is stored in self._clone_dir.
|
|
:param string packaging_utility: packaging utility as defined in test.env.yaml
|
|
"""
|
|
tempdir = tempfile.TemporaryDirectory()
|
|
args = [
|
|
"--branch",
|
|
self._branch,
|
|
f'rpms/{self._module_name}',
|
|
tempdir.name
|
|
]
|
|
packaging_util = Command(packaging_utility)
|
|
packaging_util("clone", *args)
|
|
self._clone_dir = tempdir
|
|
|
|
def bump(self):
|
|
"""Create a "bump" commit and push it in git"""
|
|
args = [
|
|
"--allow-empty",
|
|
"-m",
|
|
"Bump"
|
|
]
|
|
with pushd(self._clone_dir.name):
|
|
git("commit", *args)
|
|
git("push")
|