Merge #1661 MBSResolver - caching, tests, fixes

This commit is contained in:
Brendan Reilly
2022-05-03 16:42:22 +00:00
11 changed files with 861 additions and 452 deletions

View File

@@ -82,6 +82,9 @@ class LocalBuildConfiguration(BaseConfiguration):
RPMS_ALLOW_REPOSITORY = True
MODULES_ALLOW_REPOSITORY = True
# Match the Fedora server-side configuration
ALLOW_ONLY_COMPATIBLE_BASE_MODULES = False
# Celery tasks will be executed locally for local builds
CELERY_TASK_ALWAYS_EAGER = True
@@ -440,8 +443,12 @@ class Config(object):
"type": bool,
"default": True,
"desc": "When True, only modules built on top of compatible base modules are "
"considered by MBS as possible buildrequirement. When False, modules "
"built against any base module stream can be used as a buildrequire.",
"considered by MBS as possible buildrequirement or as a reuse candidate. "
"When False, modules built against any base module stream sharing a "
"virtual stream can be used as a buildrequire, but only modules built "
"against the exact same base module stream are considered as candidates "
"for reuse. Setting this to True requires base module streams to be of the "
" form: <prefix><x>.<y>.<z>.",
},
"base_module_stream_exclusions": {
"type": list,

View File

@@ -21,6 +21,10 @@ class UnprocessableEntity(ValueError):
pass
class StreamNotXyz(UnprocessableEntity):
pass
class Conflict(ValueError):
pass

View File

@@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
from __future__ import absolute_import, print_function
from collections import defaultdict
from functools import wraps
import getpass
import logging
@@ -16,7 +17,7 @@ from module_build_service.builder.MockModuleBuilder import (
import_builds_from_local_dnf_repos, load_local_builds
)
from module_build_service.common import conf, models
from module_build_service.common.errors import StreamAmbigous
from module_build_service.common.errors import StreamAmbigous, StreamNotXyz
from module_build_service.common.logger import level_flags
from module_build_service.common.utils import load_mmd_file, import_mmd
import module_build_service.scheduler.consumer
@@ -103,11 +104,33 @@ def import_module(mmd_file):
import_mmd(db.session, mmd)
def collect_dep_overrides(overrides):
collected = defaultdict(list)
for value in overrides:
parts = value.split(":")
if len(parts) != 2:
raise ValueError("dependency overrides must be in the form name:stream")
name, stream = parts
collected[name].append(stream)
return collected
@manager.option("--stream", action="store", dest="stream")
@manager.option("--file", action="store", dest="yaml_file")
@manager.option("--srpm", action="append", default=[], dest="srpms", metavar="SRPM")
@manager.option("--skiptests", action="store_true", dest="skiptests")
@manager.option("--offline", action="store_true", dest="offline")
@manager.option(
'--buildrequires', action='append', metavar='name:stream',
dest='buildrequires', default=[],
help='Buildrequires to override in the form of "name:stream"'
)
@manager.option(
'--requires', action='append', metavar='name:stream',
dest='requires', default=[],
help='Requires to override in the form of "name:stream"'
)
@manager.option("-d", "--debug", action="store_true", dest="log_debug")
@manager.option("-l", "--add-local-build", action="append", default=None, dest="local_build_nsvs")
@manager.option("-s", "--set-stream", action="append", default=[], dest="default_streams")
@@ -125,6 +148,8 @@ def build_module_locally(
offline=False,
platform_repofiles=None,
platform_id=None,
requires=None,
buildrequires=None,
log_debug=False,
):
""" Performs local module build using Mock
@@ -163,7 +188,9 @@ def build_module_locally(
params = {
"local_build": True,
"default_streams": dict(ns.split(":") for ns in default_streams)
"default_streams": dict(ns.split(":") for ns in default_streams),
"require_overrides": collect_dep_overrides(requires),
"buildrequire_overrides": collect_dep_overrides(buildrequires),
}
if srpms:
params["srpms"] = srpms
@@ -190,7 +217,11 @@ def build_module_locally(
except StreamAmbigous as e:
logging.error(str(e))
logging.error("Use '-s module_name:module_stream' to choose the stream")
return
return 1
except StreamNotXyz as e:
logging.error(str(e))
logging.error("Use '--buildrequires name:stream' to override the base module stream")
return 1
module_build_ids = [build.id for build in module_builds]

View File

@@ -6,7 +6,7 @@ import sqlalchemy
from sqlalchemy.orm import aliased
from module_build_service.common import log, models
from module_build_service.common.errors import UnprocessableEntity
from module_build_service.common.errors import StreamNotXyz, UnprocessableEntity
from module_build_service.common.utils import load_mmd
from module_build_service.resolver.base import GenericResolver
@@ -23,11 +23,10 @@ class DBResolver(GenericResolver):
self.config = config
def get_module(
self, name, stream, version, context,
state=models.BUILD_STATES["ready"], strict=False
self, name, stream, version, context, strict=False
):
mb = models.ModuleBuild.get_build_from_nsvc(
self.db_session, name, stream, version, context, state=state)
self.db_session, name, stream, version, context, state=models.BUILD_STATES["ready"])
if mb:
return mb.extended_json(self.db_session)
@@ -111,26 +110,31 @@ class DBResolver(GenericResolver):
:param stream_version_lte: If True, the compatible streams are limited
by the stream version computed from `stream`. If False, even the
modules with higher stream version are returned.
:param virtual_streams: List of virtual streams. If set, also modules
with incompatible stream version are returned in case they share
one of the virtual streams.
:param virtual_streams: List of virtual streams.
Modules are returned if they share one of the virtual streams.
:param states: List of states the returned compatible modules should
be in.
:return list: List of Modulemd objects.
"""
if not virtual_streams:
raise RuntimeError("Virtual stream list must not be empty")
name = base_module_mmd.get_module_name()
stream = base_module_mmd.get_stream_name()
builds = []
stream_version = None
if stream_version_lte:
stream_in_xyz_format = len(str(models.ModuleBuild.get_stream_version(
stream, right_pad=False))) >= 5
if stream_in_xyz_format:
stream_version = models.ModuleBuild.get_stream_version(stream)
else:
log.warning(
"Cannot get compatible base modules, because stream_version_lte is used, "
"but stream %r is not in x.y.z format." % stream)
if not stream_in_xyz_format:
raise StreamNotXyz(
"Cannot get compatible base modules, because stream of resolved "
"base module %s:%s is not in x.y.z format." % (
base_module_mmd.get_module_name(), stream
))
stream_version = models.ModuleBuild.get_stream_version(stream)
builds = models.ModuleBuild.get_last_builds_in_stream_version_lte(
self.db_session, name, stream_version, virtual_streams, states)

View File

@@ -5,11 +5,11 @@
from __future__ import absolute_import
import logging
import kobo.rpmlib
import dogpile.cache
from module_build_service.common import models
from module_build_service.common.config import conf
from module_build_service.common.errors import UnprocessableEntity
from module_build_service.common.errors import StreamNotXyz, UnprocessableEntity
from module_build_service.common.request_utils import requests_session
from module_build_service.common.utils import load_mmd, import_mmd
from module_build_service.resolver.KojiResolver import KojiResolver
@@ -17,14 +17,31 @@ from module_build_service.resolver.KojiResolver import KojiResolver
log = logging.getLogger()
def _canonicalize_state(state):
if isinstance(state, int):
return models.INVERSE_BUILD_STATES[state]
else:
return state
def _canonicalize_states(states):
if states:
result = sorted({_canonicalize_state(s) for s in states})
if result == ["ready"]:
return None
else:
return result
class MBSResolver(KojiResolver):
backend = "mbs"
region = dogpile.cache.make_region().configure("dogpile.cache.memory")
def __init__(self, db_session, config):
self.db_session = db_session
self.mbs_prod_url = config.mbs_url
self._generic_error = "Failed to query MBS with query %r returned HTTP status %s"
def _query_from_nsvc(self, name, stream, version=None, context=None, states=None):
"""
@@ -38,44 +55,65 @@ class MBSResolver(KojiResolver):
states = states or ["ready"]
query = {
"name": name,
"stream": stream,
"state": states,
"verbose": True,
"order_desc_by": "version",
}
if stream is not None:
query["stream"] = stream
if version is not None:
query["version"] = str(version)
if context is not None:
query["context"] = context
return query
def _get_modules(
self, name, stream, version=None, context=None, states=None, strict=False, **kwargs
@region.cache_on_arguments()
def _get_module_by_nsvc(
self, name, stream, version, context
):
"""Query and return modules from MBS with specific info
:param str name: module's name.
:param str stream: module's stream.
:kwarg str version: a string or int of the module's version. When None,
latest version will be returned.
:kwarg str context: module's context. Optional.
:kwarg str state: module's state. Defaults to ``ready``.
:kwarg bool strict: Normally this function returns None if no module can be
found. If strict=True, then an UnprocessableEntity is raised.
:return: final list of module_info which pass repoclosure
:rtype: list[dict]
:raises UnprocessableEntity: if no modules are found and ``strict`` is True.
"""
Query a single module from MBS
:param str name: Name of the module to query.
:param str stream: Stream of the module to query.
:param str version/int: Version of the module to query.
:param str context: Context of the module to query.
"""
query = self._query_from_nsvc(name, stream, version, context)
query['per_page'] = 5
query['page'] = 1
res = requests_session.get(self.mbs_prod_url, params=query)
res.raise_for_status()
data = res.json()
modules = data["items"]
if modules:
return modules[0]
else:
return None
# Args must exactly match call in _get_modules; newer versions of dogpile.cache
# have dogpile.cache.util.kwarg_function_key_generator which could help
@region.cache_on_arguments()
def _get_modules_with_cache(
self, name, stream, version, context,
states, base_module_br, virtual_stream, stream_version_lte
):
query = self._query_from_nsvc(name, stream, version, context, states)
query["page"] = 1
query["per_page"] = 10
query.update(kwargs)
query["per_page"] = 5
if virtual_stream is not None:
query["virtual_stream"] = virtual_stream
if stream_version_lte is not None:
query["stream_version_lte"] = stream_version_lte
if base_module_br is not None:
query["base_module_br"] = base_module_br
modules = []
while True:
res = requests_session.get(self.mbs_prod_url, params=query)
if not res.ok:
raise RuntimeError(self._generic_error % (query, res.status_code))
res.raise_for_status()
data = res.json()
modules_per_page = data["items"]
@@ -84,23 +122,70 @@ class MBSResolver(KojiResolver):
if not data["meta"]["next"]:
break
if version is None and stream_version_lte is None:
# Stop querying when we've gotten a different version
if modules_per_page[-1]["version"] != modules[0]["version"]:
break
query["page"] += 1
# Error handling
if not modules:
if strict:
raise UnprocessableEntity("Failed to find module in MBS %r" % query)
else:
return modules
if version is None and "stream_version_lte" not in kwargs:
if version is None and stream_version_lte is None:
# Only return the latest version
return [m for m in modules if m["version"] == modules[0]["version"]]
results = [m for m in modules if m["version"] == modules[0]["version"]]
else:
results = modules
for m in results:
# We often come back and query details again for the module builds we return
# using the retrieved contexts, so prime the cache for a single-module queries
self._get_module_by_nsvc.set(
m, self, m["name"], m["stream"], m["version"], m["context"]
)
return results
def _get_modules(
self, name, stream, version=None, context=None, states=None,
base_module_br=None, virtual_stream=None, stream_version_lte=None,
strict=False,
):
"""Query and return modules from MBS with specific info
:param str name: module's name.
:param str stream: module's stream.
:kwarg str version: a string or int of the module's version. When None,
latest version will be returned.
:kwarg str context: module's context. Optional.
:kwarg str states: states for modules. Defaults to ``["ready"]``.
:kwarg str base_module_br: if set, restrict results to modules requiring
this base_module nsvc
:kwarg list virtual_stream: if set, limit to modules for the given virtual_stream
:kwarg float stream_version_lte: If set, stream must be less than this version
:kwarg bool strict: Normally this function returns None if no module can be
found. If strict=True, then an UnprocessableEntity is raised.
:return: final list of module_info which pass repoclosure
:rtype: list[dict]
"""
modules = self._get_modules_with_cache(
name, stream, version, context,
states, base_module_br, virtual_stream, stream_version_lte
)
if not modules and strict:
raise UnprocessableEntity("Failed to find module in MBS %s:%s:%s:%s" %
(name, stream, version, context))
else:
return modules
def get_module(self, name, stream, version, context, states=None, strict=False):
rv = self._get_modules(name, stream, version, context, states, strict)
def get_module(self, name, stream, version, context, strict=False):
if version and context:
rv = self._get_module_by_nsvc(name, stream, version, context)
if strict and rv is None:
raise UnprocessableEntity("Failed to find module in MBS")
return rv
rv = self._get_modules(name, stream, version, context, strict=strict)
if rv:
return rv[0]
@@ -114,8 +199,7 @@ class MBSResolver(KojiResolver):
query = {"page": 1, "per_page": 1, "short": True}
query.update(kwargs)
res = requests_session.get(self.mbs_prod_url, params=query)
if not res.ok:
raise RuntimeError(self._generic_error % (query, res.status_code))
res.raise_for_status()
data = res.json()
return data["meta"]["total"]
@@ -138,23 +222,37 @@ class MBSResolver(KojiResolver):
"virtual_stream": virtual_stream,
}
res = requests_session.get(self.mbs_prod_url, params=query)
if not res.ok:
raise RuntimeError(self._generic_error % (query, res.status_code))
res.raise_for_status()
data = res.json()
if data["items"]:
return load_mmd(data["items"][0]["modulemd"])
def _modules_to_modulemds(self, modules, strict):
if not modules:
return []
mmds = []
for module in modules:
yaml = module["modulemd"]
if not yaml:
if strict:
raise UnprocessableEntity(
"Failed to find modulemd entry in MBS for %r" % module)
else:
log.warning("Failed to find modulemd entry in MBS for %r", module)
continue
mmds.append(load_mmd(yaml))
return mmds
def get_module_modulemds(
self,
name,
stream,
version=None,
context=None,
strict=False,
stream_version_lte=False,
virtual_streams=None,
states=None,
strict=False
):
"""
Gets the module modulemds from the resolver.
@@ -166,48 +264,15 @@ class MBSResolver(KojiResolver):
be returned.
:kwarg strict: Normally this function returns [] if no module can be
found. If strict=True, then a UnprocessableEntity is raised.
:kwarg stream_version_lte: If True and if the `stream` can be transformed to
"stream version", the returned list will include all the modules with stream version
less than or equal the stream version computed from `stream`.
:kwarg virtual_streams: a list of the virtual streams to filter on. The filtering uses "or"
logic. When falsy, no filtering occurs.
:return: List of Modulemd metadata instances matching the query
"""
yaml = None
local_modules = models.ModuleBuild.local_modules(self.db_session, name, stream)
if local_modules:
return [m.mmd() for m in local_modules]
extra_args = {}
if stream_version_lte and (
len(str(models.ModuleBuild.get_stream_version(stream, right_pad=False))) >= 5
):
stream_version = models.ModuleBuild.get_stream_version(stream)
extra_args["stream_version_lte"] = stream_version
modules = self._get_modules(name, stream, version, context, strict=strict)
if virtual_streams:
extra_args["virtual_stream"] = virtual_streams
modules = self._get_modules(name, stream, version, context, strict=strict, states=states,
**extra_args)
if not modules:
return []
mmds = []
for module in modules:
if module:
yaml = module["modulemd"]
if not yaml:
if strict:
raise UnprocessableEntity(
"Failed to find modulemd entry in MBS for %r" % module)
else:
return None
mmds.append(load_mmd(yaml))
return mmds
return self._modules_to_modulemds(modules, strict)
def get_compatible_base_module_modulemds(
self, base_module_mmd, stream_version_lte, virtual_streams, states
@@ -225,18 +290,38 @@ class MBSResolver(KojiResolver):
:param stream_version_lte: If True, the compatible streams are limited
by the stream version computed from `stream`. If False, even the
modules with higher stream version are returned.
:param virtual_streams: List of virtual streams. If set, also modules
with incompatible stream version are returned in case they share
one of the virtual streams.
:param virtual_streams: List of virtual streams.
Modules are returned if they share one of the virtual streams.
:param states: List of states the returned compatible modules should
be in.
:return list: List of Modulemd objects.
"""
if not virtual_streams:
raise RuntimeError("Virtual stream list must not be empty")
name = base_module_mmd.get_module_name()
stream = base_module_mmd.get_stream_name()
return self.get_module_modulemds(
name, stream, stream_version_lte=stream_version_lte, virtual_streams=virtual_streams,
states=states)
extra_args = {}
if stream_version_lte:
stream = base_module_mmd.get_stream_name()
stream_in_xyz_format = len(str(models.ModuleBuild.get_stream_version(
stream, right_pad=False))) >= 5
if not stream_in_xyz_format:
raise StreamNotXyz(
"Cannot get compatible base modules, because stream of resolved "
"base module %s:%s is not in x.y.z format." % (
base_module_mmd.get_module_name(), stream
))
stream_version = models.ModuleBuild.get_stream_version(stream)
extra_args["stream_version_lte"] = stream_version
extra_args["virtual_stream"] = virtual_streams
extra_args["states"] = _canonicalize_states(states)
modules = self._get_modules(name, None, **extra_args)
return self._modules_to_modulemds(modules, False)
def get_buildrequired_modulemds(self, name, stream, base_module_mmd):
"""
@@ -269,7 +354,7 @@ class MBSResolver(KojiResolver):
return ret
else:
modules = self._get_modules(
name, stream, strict=False, base_module_br=base_module_mmd.get_nsvc())
name, stream, base_module_br=base_module_mmd.get_nsvc())
return [load_mmd(module["modulemd"]) for module in modules]
def resolve_profiles(self, mmd, keys):
@@ -304,7 +389,7 @@ class MBSResolver(KojiResolver):
continue
# Find the dep in the built modules in MBS
modules = self._get_modules(
module = self.get_module(
module_name,
module_info["stream"],
module_info["version"],
@@ -312,14 +397,13 @@ class MBSResolver(KojiResolver):
strict=True,
)
for module in modules:
yaml = module["modulemd"]
dep_mmd = load_mmd(yaml)
# Take note of what rpms are in this dep's profile.
for key in keys:
profile = dep_mmd.get_profile(key)
if profile:
results[key] |= set(profile.get_rpms())
yaml = module["modulemd"]
dep_mmd = load_mmd(yaml)
# Take note of what rpms are in this dep's profile.
for key in keys:
profile = dep_mmd.get_profile(key)
if profile:
results[key] |= set(profile.get_rpms())
# Return the union of all rpms in all profiles of the given keys.
return results
@@ -365,12 +449,15 @@ class MBSResolver(KojiResolver):
else:
queried_module = self.get_module(name, stream, version, context, strict=strict)
yaml = queried_module["modulemd"]
queried_mmd = load_mmd(yaml)
if yaml:
queried_mmd = load_mmd(yaml)
else:
queried_mmd = None
if not queried_mmd or "buildrequires" not in queried_mmd.get_xmd().get("mbs", {}):
raise RuntimeError(
'The module "{0!r}" did not contain its modulemd or did not have '
"its xmd attribute filled out in MBS".format(queried_mmd)
"its xmd attribute filled out in MBS".format(queried_module)
)
buildrequires = queried_mmd.get_xmd()["mbs"]["buildrequires"]
@@ -380,26 +467,23 @@ class MBSResolver(KojiResolver):
self.db_session, name, details["stream"])
if local_modules:
for m in local_modules:
# If the buildrequire is a meta-data only module with no Koji tag set, then just
# skip it
if m.koji_tag is None:
continue
module_tags[m.koji_tag] = m.mmd()
continue
if "context" not in details:
details["context"] = models.DEFAULT_MODULE_CONTEXT
modules = self._get_modules(
m = self.get_module(
name, details["stream"], details["version"], details["context"], strict=True)
for m in modules:
if m["koji_tag"] in module_tags:
continue
# If the buildrequire is a meta-data only module with no Koji tag set, then just
# skip it
if m["koji_tag"] is None:
continue
module_tags.setdefault(m["koji_tag"], [])
module_tags[m["koji_tag"]].append(load_mmd(m["modulemd"]))
if m["koji_tag"] in module_tags:
continue
# If the buildrequire is a meta-data only module with no Koji tag set, then just
# skip it
if m["koji_tag"] is None:
continue
module_tags.setdefault(m["koji_tag"], [])
module_tags[m["koji_tag"]].append(load_mmd(m["modulemd"]))
return module_tags
@@ -448,7 +532,6 @@ class MBSResolver(KojiResolver):
commit_hash = None
version = None
filtered_rpms = []
module = self.get_module(
module_name, module_stream, module_version, module_context, strict=True
)
@@ -457,20 +540,6 @@ class MBSResolver(KojiResolver):
if mmd.get_xmd().get("mbs", {}).get("commit"):
commit_hash = mmd.get_xmd()["mbs"]["commit"]
# Find out the particular NVR of filtered packages
if "rpms" in module and mmd.get_rpm_filters():
for rpm in module["rpms"]:
nvr = kobo.rpmlib.parse_nvra(rpm)
# If the package is not filtered, continue
if not nvr["name"] in mmd.get_rpm_filters():
continue
# If the nvr is already in filtered_rpms, continue
nvr = kobo.rpmlib.make_nvr(nvr, force_epoch=True)
if nvr in filtered_rpms:
continue
filtered_rpms.append(nvr)
if module.get("version"):
version = module["version"]
@@ -481,11 +550,10 @@ class MBSResolver(KojiResolver):
"version": str(version),
"context": module["context"],
"koji_tag": module["koji_tag"],
"filtered_rpms": filtered_rpms,
}
else:
raise RuntimeError(
'The module "{0}" didn\'t contain either a commit hash or a'
'The module "{0}" didn\'t contain both a commit hash and a'
" version in MBS".format(module_name)
)
# If the module is a base module, then import it in the database so that entries in

View File

@@ -78,7 +78,7 @@ class GenericResolver(six.with_metaclass(ABCMeta)):
return False
@abstractmethod
def get_module(self, name, stream, version, context, state="ready", strict=False):
def get_module(self, name, stream, version, context, strict=False):
raise NotImplementedError()
@abstractmethod

View File

@@ -83,7 +83,9 @@ def get_reusable_module(module):
#
# 1) The `conf.allow_only_compatible_base_modules` is False. This means that MBS should
# not try to find any compatible base modules in its DB and simply use the buildrequired
# base module as it is.
# base module as it is. (*NOTE* This is different than the meaning of
# allow_only_compatible_base_modules=False when looking up build requirements - where
# it means to accept any module buildrequiring a base modul sharing a virtual stream.)
# 2) The `conf.allow_only_compatible_base_modules` is True and DBResolver is used. This means
# that MBS should try to find the compatible modules using its database.
# The `get_base_module_mmds` finds out the list of compatible modules and returns mmds of