modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-27 12:12:22 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge #1435 Add initial code for KojiResolver class.

Browse Source
This commit is contained in:
Jan Kaluža
2019-09-30 06:25:55 +00:00
parent 8e3ebe5df2 a7540452cd
commit f6c2a88cc0
9 changed files with 393 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
module_build_service/resolver/DBResolver.py
View File

@@ -155,19 +155,19 @@ class DBResolver(GenericResolver):
return [build.mmd() for build in builds]
def get_buildrequired_modulemds(self, name, stream, base_module_nsvc):
def get_buildrequired_modulemds(self, name, stream, base_module_mmd):
"""
Returns modulemd metadata of all module builds with `name` and `stream` buildrequiring
base module defined by `base_module_nsvc` NSVC.
base module defined by `base_module_mmd` NSVC.
:param str name: Name of module to return.
:param str stream: Stream of module to return.
:param str base_module_nsvc: NSVC of base module which must be buildrequired by returned
:param Modulemd base_module_mmd: NSVC of base module which must be buildrequired by returned
modules.
:rtype: list
:return: List of modulemd metadata.
"""
log.debug("Looking for %s:%s buildrequiring %s", name, stream, base_module_nsvc)
log.debug("Looking for %s:%s buildrequiring %s", name, stream, base_module_mmd.get_nsvc())
query = self.db_session.query(models.ModuleBuild)
query = query.filter_by(name=name, stream=stream, state=models.BUILD_STATES["ready"])
@@ -182,8 +182,8 @@ class DBResolver(GenericResolver):
query = query.join(mb_to_br, mb_to_br.c.module_id == models.ModuleBuild.id).join(
module_br_alias, mb_to_br.c.module_buildrequire_id == module_br_alias.id)
# Get only modules buildrequiring particular base_module_nsvc
n, s, v, c = base_module_nsvc.split(":")
# Get only modules buildrequiring particular base_module_mmd
n, s, v, c = base_module_mmd.get_nsvc().split(":")
query = query.filter(
module_br_alias.name == n,
module_br_alias.stream == s,

162
module_build_service/resolver/KojiResolver.py Normal file
View File

@@ -0,0 +1,162 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019 Red Hat, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Written by Jan Kaluza <jkaluza@redhat.com>
from itertools import groupby
from module_build_service.resolver.DBResolver import DBResolver
from module_build_service import conf, models, log
class KojiResolver(DBResolver):
"""
Resolver using Koji server running in infrastructure.
"""
backend = "koji"
def _filter_inherited(self, koji_session, module_builds, top_tag, event):
"""
Look at the tag inheritance and keep builds only from the topmost tag.
For example, we have "foo:bar:1" and "foo:bar:2" builds. We also have "foo-tag" which
inherits "foo-parent-tag". The "foo:bar:1" is tagged in the "foo-tag". The "foo:bar:2"
is tagged in the "foo-parent-tag".
In this case, this function filters out the foo:bar:2, because "foo:bar:1" is tagged
lower in the inheritance tree in the "foo-tag".
For normal RPMs, using latest=True for listTagged() call, Koji would automatically do
this, but it does not understand streams, so we have to reimplement it here.
:param KojiSession koji_session: Koji session.
:param list module_builds: List of builds as returned by KojiSession.listTagged method.
:param str top_tag: The top Koji tag.
:param dict event: Koji event defining the time at which the `module_builds` have been
fetched.
:return list: Filtered list of builds.
"""
inheritance = [
tag["name"] for tag in koji_session.getFullInheritance(top_tag, event=event["id"])
]
def keyfunc(mb):
return (mb["name"], mb["version"])
result = []
# Group modules by Name-Stream
for _, builds in groupby(sorted(module_builds, key=keyfunc), keyfunc):
builds = list(builds)
# For each N-S combination find out which tags it's in
available_in = set(build["tag_name"] for build in builds)
# And find out which is the topmost tag
for tag in [top_tag] + inheritance:
if tag in available_in:
break
# And keep only builds from that topmost tag
result.extend(build for build in builds if build["tag_name"] == tag)
return result
def get_buildrequired_modulemds(self, name, stream, base_module_mmd):
"""
Returns modulemd metadata of all module builds with `name` and `stream` which are tagged
in the Koji tag defined in `base_module_mmd`.
:param str name: Name of module to return.
:param str stream: Stream of module to return.
:param Modulemd base_module_mmd: Base module metadata.
:return list: List of modulemd metadata.
"""
# Get the `koji_tag_with_modules`. If the `koji_tag_with_modules` is not configured for
# the base module, fallback to DBResolver.
tag = base_module_mmd.get_xmd().get("mbs", {}).get("koji_tag_with_modules")
if not tag:
log.info(
"The %s does not define 'koji_tag_with_modules'. Falling back to DBResolver." % (
base_module_mmd.get_nsvc()))
return DBResolver.get_buildrequired_modulemds(self, name, stream, base_module_mmd)
# Create KojiSession. We need to import here because of circular dependencies.
from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder
koji_session = KojiModuleBuilder.get_session(conf, login=False)
event = koji_session.getLastEvent()
# List all the modular builds in the modular Koji tag.
# We cannot use latest=True here, because we need to get all the
# available streams of all modules. The stream is represented as
# "version" in Koji build and with latest=True, Koji would return
# only builds with the highest version.
# We also cannot ask for particular `stream`, because Koji does not support that.
module_builds = koji_session.listTagged(
tag, inherit=True, type="module", package=name, event=event["id"])
# Filter out different streams
normalized_stream = stream.replace("-", "_")
module_builds = [b for b in module_builds if b["version"] == normalized_stream]
# Filter out builds inherited from non-top tag
module_builds = self._filter_inherited(koji_session, module_builds, tag, event)
# Find the latest builds of all modules. This does the following:
# - Sorts the module_builds descending by Koji NVR (which maps to NSV
# for modules). Split release into modular version and context, and
# treat version as numeric.
# - Groups the sorted module_builds by NV (NS in modular world).
# In each resulting `ns_group`, the first item is actually build
# with the latest version (because the list is still sorted by NVR).
# - Groups the `ns_group` again by "release" ("version" in modular
# world) to just get all the "contexts" of the given NSV. This is
# stored in `nsv_builds`.
# - The `nsv_builds` contains the builds representing all the contexts
# of the latest version for give name-stream, so add them to
# `latest_builds`.
def _key(build):
ver, ctx = build["release"].split(".", 1)
return build["name"], build["version"], int(ver), ctx
latest_builds = []
module_builds = sorted(module_builds, key=_key, reverse=True)
for _, ns_builds in groupby(
module_builds, key=lambda x: ":".join([x["name"], x["version"]])):
for _, nsv_builds in groupby(
ns_builds, key=lambda x: x["release"].split(".")[0]):
latest_builds += list(nsv_builds)
break
# For each latest module build, find the matching ModuleBuild and store its modulemd
# in `mmds`.
mmds = []
for build in latest_builds:
version, context = build["release"].split(".")
module = models.ModuleBuild.get_build_from_nsvc(
self.db_session, name, stream, version, context)
if not module:
raise ValueError(
"Module %s is tagged in the %s Koji tag, but does not exist "
"in MBS DB." % (":".join([name, stream, version, context]), tag))
mmds.append(module.mmd())
return mmds

6
module_build_service/resolver/LocalResolver.py
View File

@@ -34,12 +34,12 @@ class LocalResolver(DBResolver):
backend = "local"
def get_buildrequired_modulemds(self, name, stream, base_module_nsvc):
def get_buildrequired_modulemds(self, name, stream, base_module_mmd):
"""
Returns modulemd metadata of all module builds with `name` and `stream`.
For LocalResolver which is used only for Offline local builds,
the `base_module_nsvc` is ignored. Normally, the `base_module_nsvc is used
the `base_module_mmd` is ignored. Normally, the `base_module_mmd is used
to filter out platform:streams which are not compatible with currently used
stream version. But during offline local builds, we always have just single
platform:stream derived from PLATFORM_ID in /etc/os-release.
@@ -50,7 +50,7 @@ class LocalResolver(DBResolver):
:param str name: Name of module to return.
:param str stream: Stream of module to return.
:param str base_module_nsvc: Ignored in LocalResolver.
:param Modulemd base_module_mmd: Ignored in LocalResolver.
:rtype: list
:return: List of modulemd metadata.
"""

2
module_build_service/resolver/base.py
View File

@@ -121,7 +121,7 @@ class GenericResolver(six.with_metaclass(ABCMeta)):
raise NotImplementedError()
@abstractmethod
def get_buildrequired_modulemds(self, name, stream, base_module_nsvc, strict=False):
def get_buildrequired_modulemds(self, name, stream, base_module_mmd, strict=False):
raise NotImplementedError()
@abstractmethod

3
module_build_service/utils/mse.py
View File

@@ -189,8 +189,7 @@ def _get_mmds_from_requires(
if base_module_mmds:
for base_module_mmd in base_module_mmds:
base_module_nsvc = base_module_mmd.get_nsvc()
mmds[ns] += resolver.get_buildrequired_modulemds(name, stream, base_module_nsvc)
mmds[ns] += resolver.get_buildrequired_modulemds(name, stream, base_module_mmd)
else:
mmds[ns] = resolver.get_module_modulemds(name, stream, strict=True)
added_mmds[ns] += mmds[ns]