mirror of
https://pagure.io/fm-orchestrator.git
synced 2026-02-02 20:59:06 +08:00
Merge #1661 MBSResolver - caching, tests, fixes
This commit is contained in:
@@ -82,6 +82,9 @@ class LocalBuildConfiguration(BaseConfiguration):
|
||||
RPMS_ALLOW_REPOSITORY = True
|
||||
MODULES_ALLOW_REPOSITORY = True
|
||||
|
||||
# Match the Fedora server-side configuration
|
||||
ALLOW_ONLY_COMPATIBLE_BASE_MODULES = False
|
||||
|
||||
# Celery tasks will be executed locally for local builds
|
||||
CELERY_TASK_ALWAYS_EAGER = True
|
||||
|
||||
@@ -440,8 +443,12 @@ class Config(object):
|
||||
"type": bool,
|
||||
"default": True,
|
||||
"desc": "When True, only modules built on top of compatible base modules are "
|
||||
"considered by MBS as possible buildrequirement. When False, modules "
|
||||
"built against any base module stream can be used as a buildrequire.",
|
||||
"considered by MBS as possible buildrequirement or as a reuse candidate. "
|
||||
"When False, modules built against any base module stream sharing a "
|
||||
"virtual stream can be used as a buildrequire, but only modules built "
|
||||
"against the exact same base module stream are considered as candidates "
|
||||
"for reuse. Setting this to True requires base module streams to be of the "
|
||||
" form: <prefix><x>.<y>.<z>.",
|
||||
},
|
||||
"base_module_stream_exclusions": {
|
||||
"type": list,
|
||||
|
||||
@@ -21,6 +21,10 @@ class UnprocessableEntity(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class StreamNotXyz(UnprocessableEntity):
|
||||
pass
|
||||
|
||||
|
||||
class Conflict(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# SPDX-License-Identifier: MIT
|
||||
from __future__ import absolute_import, print_function
|
||||
from collections import defaultdict
|
||||
from functools import wraps
|
||||
import getpass
|
||||
import logging
|
||||
@@ -16,7 +17,7 @@ from module_build_service.builder.MockModuleBuilder import (
|
||||
import_builds_from_local_dnf_repos, load_local_builds
|
||||
)
|
||||
from module_build_service.common import conf, models
|
||||
from module_build_service.common.errors import StreamAmbigous
|
||||
from module_build_service.common.errors import StreamAmbigous, StreamNotXyz
|
||||
from module_build_service.common.logger import level_flags
|
||||
from module_build_service.common.utils import load_mmd_file, import_mmd
|
||||
import module_build_service.scheduler.consumer
|
||||
@@ -103,11 +104,33 @@ def import_module(mmd_file):
|
||||
import_mmd(db.session, mmd)
|
||||
|
||||
|
||||
def collect_dep_overrides(overrides):
|
||||
collected = defaultdict(list)
|
||||
for value in overrides:
|
||||
parts = value.split(":")
|
||||
if len(parts) != 2:
|
||||
raise ValueError("dependency overrides must be in the form name:stream")
|
||||
name, stream = parts
|
||||
collected[name].append(stream)
|
||||
|
||||
return collected
|
||||
|
||||
|
||||
@manager.option("--stream", action="store", dest="stream")
|
||||
@manager.option("--file", action="store", dest="yaml_file")
|
||||
@manager.option("--srpm", action="append", default=[], dest="srpms", metavar="SRPM")
|
||||
@manager.option("--skiptests", action="store_true", dest="skiptests")
|
||||
@manager.option("--offline", action="store_true", dest="offline")
|
||||
@manager.option(
|
||||
'--buildrequires', action='append', metavar='name:stream',
|
||||
dest='buildrequires', default=[],
|
||||
help='Buildrequires to override in the form of "name:stream"'
|
||||
)
|
||||
@manager.option(
|
||||
'--requires', action='append', metavar='name:stream',
|
||||
dest='requires', default=[],
|
||||
help='Requires to override in the form of "name:stream"'
|
||||
)
|
||||
@manager.option("-d", "--debug", action="store_true", dest="log_debug")
|
||||
@manager.option("-l", "--add-local-build", action="append", default=None, dest="local_build_nsvs")
|
||||
@manager.option("-s", "--set-stream", action="append", default=[], dest="default_streams")
|
||||
@@ -125,6 +148,8 @@ def build_module_locally(
|
||||
offline=False,
|
||||
platform_repofiles=None,
|
||||
platform_id=None,
|
||||
requires=None,
|
||||
buildrequires=None,
|
||||
log_debug=False,
|
||||
):
|
||||
""" Performs local module build using Mock
|
||||
@@ -163,7 +188,9 @@ def build_module_locally(
|
||||
|
||||
params = {
|
||||
"local_build": True,
|
||||
"default_streams": dict(ns.split(":") for ns in default_streams)
|
||||
"default_streams": dict(ns.split(":") for ns in default_streams),
|
||||
"require_overrides": collect_dep_overrides(requires),
|
||||
"buildrequire_overrides": collect_dep_overrides(buildrequires),
|
||||
}
|
||||
if srpms:
|
||||
params["srpms"] = srpms
|
||||
@@ -190,7 +217,11 @@ def build_module_locally(
|
||||
except StreamAmbigous as e:
|
||||
logging.error(str(e))
|
||||
logging.error("Use '-s module_name:module_stream' to choose the stream")
|
||||
return
|
||||
return 1
|
||||
except StreamNotXyz as e:
|
||||
logging.error(str(e))
|
||||
logging.error("Use '--buildrequires name:stream' to override the base module stream")
|
||||
return 1
|
||||
|
||||
module_build_ids = [build.id for build in module_builds]
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import sqlalchemy
|
||||
from sqlalchemy.orm import aliased
|
||||
|
||||
from module_build_service.common import log, models
|
||||
from module_build_service.common.errors import UnprocessableEntity
|
||||
from module_build_service.common.errors import StreamNotXyz, UnprocessableEntity
|
||||
from module_build_service.common.utils import load_mmd
|
||||
from module_build_service.resolver.base import GenericResolver
|
||||
|
||||
@@ -23,11 +23,10 @@ class DBResolver(GenericResolver):
|
||||
self.config = config
|
||||
|
||||
def get_module(
|
||||
self, name, stream, version, context,
|
||||
state=models.BUILD_STATES["ready"], strict=False
|
||||
self, name, stream, version, context, strict=False
|
||||
):
|
||||
mb = models.ModuleBuild.get_build_from_nsvc(
|
||||
self.db_session, name, stream, version, context, state=state)
|
||||
self.db_session, name, stream, version, context, state=models.BUILD_STATES["ready"])
|
||||
if mb:
|
||||
return mb.extended_json(self.db_session)
|
||||
|
||||
@@ -111,26 +110,31 @@ class DBResolver(GenericResolver):
|
||||
:param stream_version_lte: If True, the compatible streams are limited
|
||||
by the stream version computed from `stream`. If False, even the
|
||||
modules with higher stream version are returned.
|
||||
:param virtual_streams: List of virtual streams. If set, also modules
|
||||
with incompatible stream version are returned in case they share
|
||||
one of the virtual streams.
|
||||
:param virtual_streams: List of virtual streams.
|
||||
Modules are returned if they share one of the virtual streams.
|
||||
:param states: List of states the returned compatible modules should
|
||||
be in.
|
||||
:return list: List of Modulemd objects.
|
||||
"""
|
||||
if not virtual_streams:
|
||||
raise RuntimeError("Virtual stream list must not be empty")
|
||||
|
||||
name = base_module_mmd.get_module_name()
|
||||
stream = base_module_mmd.get_stream_name()
|
||||
builds = []
|
||||
|
||||
stream_version = None
|
||||
if stream_version_lte:
|
||||
stream_in_xyz_format = len(str(models.ModuleBuild.get_stream_version(
|
||||
stream, right_pad=False))) >= 5
|
||||
if stream_in_xyz_format:
|
||||
stream_version = models.ModuleBuild.get_stream_version(stream)
|
||||
else:
|
||||
log.warning(
|
||||
"Cannot get compatible base modules, because stream_version_lte is used, "
|
||||
"but stream %r is not in x.y.z format." % stream)
|
||||
if not stream_in_xyz_format:
|
||||
raise StreamNotXyz(
|
||||
"Cannot get compatible base modules, because stream of resolved "
|
||||
"base module %s:%s is not in x.y.z format." % (
|
||||
base_module_mmd.get_module_name(), stream
|
||||
))
|
||||
stream_version = models.ModuleBuild.get_stream_version(stream)
|
||||
|
||||
builds = models.ModuleBuild.get_last_builds_in_stream_version_lte(
|
||||
self.db_session, name, stream_version, virtual_streams, states)
|
||||
|
||||
|
||||
@@ -5,11 +5,11 @@
|
||||
from __future__ import absolute_import
|
||||
import logging
|
||||
|
||||
import kobo.rpmlib
|
||||
import dogpile.cache
|
||||
|
||||
from module_build_service.common import models
|
||||
from module_build_service.common.config import conf
|
||||
from module_build_service.common.errors import UnprocessableEntity
|
||||
from module_build_service.common.errors import StreamNotXyz, UnprocessableEntity
|
||||
from module_build_service.common.request_utils import requests_session
|
||||
from module_build_service.common.utils import load_mmd, import_mmd
|
||||
from module_build_service.resolver.KojiResolver import KojiResolver
|
||||
@@ -17,14 +17,31 @@ from module_build_service.resolver.KojiResolver import KojiResolver
|
||||
log = logging.getLogger()
|
||||
|
||||
|
||||
def _canonicalize_state(state):
|
||||
if isinstance(state, int):
|
||||
return models.INVERSE_BUILD_STATES[state]
|
||||
else:
|
||||
return state
|
||||
|
||||
|
||||
def _canonicalize_states(states):
|
||||
if states:
|
||||
result = sorted({_canonicalize_state(s) for s in states})
|
||||
if result == ["ready"]:
|
||||
return None
|
||||
else:
|
||||
return result
|
||||
|
||||
|
||||
class MBSResolver(KojiResolver):
|
||||
|
||||
backend = "mbs"
|
||||
|
||||
region = dogpile.cache.make_region().configure("dogpile.cache.memory")
|
||||
|
||||
def __init__(self, db_session, config):
|
||||
self.db_session = db_session
|
||||
self.mbs_prod_url = config.mbs_url
|
||||
self._generic_error = "Failed to query MBS with query %r returned HTTP status %s"
|
||||
|
||||
def _query_from_nsvc(self, name, stream, version=None, context=None, states=None):
|
||||
"""
|
||||
@@ -38,44 +55,65 @@ class MBSResolver(KojiResolver):
|
||||
states = states or ["ready"]
|
||||
query = {
|
||||
"name": name,
|
||||
"stream": stream,
|
||||
"state": states,
|
||||
"verbose": True,
|
||||
"order_desc_by": "version",
|
||||
}
|
||||
if stream is not None:
|
||||
query["stream"] = stream
|
||||
if version is not None:
|
||||
query["version"] = str(version)
|
||||
if context is not None:
|
||||
query["context"] = context
|
||||
return query
|
||||
|
||||
def _get_modules(
|
||||
self, name, stream, version=None, context=None, states=None, strict=False, **kwargs
|
||||
@region.cache_on_arguments()
|
||||
def _get_module_by_nsvc(
|
||||
self, name, stream, version, context
|
||||
):
|
||||
"""Query and return modules from MBS with specific info
|
||||
|
||||
:param str name: module's name.
|
||||
:param str stream: module's stream.
|
||||
:kwarg str version: a string or int of the module's version. When None,
|
||||
latest version will be returned.
|
||||
:kwarg str context: module's context. Optional.
|
||||
:kwarg str state: module's state. Defaults to ``ready``.
|
||||
:kwarg bool strict: Normally this function returns None if no module can be
|
||||
found. If strict=True, then an UnprocessableEntity is raised.
|
||||
:return: final list of module_info which pass repoclosure
|
||||
:rtype: list[dict]
|
||||
:raises UnprocessableEntity: if no modules are found and ``strict`` is True.
|
||||
"""
|
||||
Query a single module from MBS
|
||||
|
||||
:param str name: Name of the module to query.
|
||||
:param str stream: Stream of the module to query.
|
||||
:param str version/int: Version of the module to query.
|
||||
:param str context: Context of the module to query.
|
||||
"""
|
||||
query = self._query_from_nsvc(name, stream, version, context)
|
||||
query['per_page'] = 5
|
||||
query['page'] = 1
|
||||
|
||||
res = requests_session.get(self.mbs_prod_url, params=query)
|
||||
res.raise_for_status()
|
||||
|
||||
data = res.json()
|
||||
modules = data["items"]
|
||||
if modules:
|
||||
return modules[0]
|
||||
else:
|
||||
return None
|
||||
|
||||
# Args must exactly match call in _get_modules; newer versions of dogpile.cache
|
||||
# have dogpile.cache.util.kwarg_function_key_generator which could help
|
||||
@region.cache_on_arguments()
|
||||
def _get_modules_with_cache(
|
||||
self, name, stream, version, context,
|
||||
states, base_module_br, virtual_stream, stream_version_lte
|
||||
):
|
||||
query = self._query_from_nsvc(name, stream, version, context, states)
|
||||
query["page"] = 1
|
||||
query["per_page"] = 10
|
||||
query.update(kwargs)
|
||||
query["per_page"] = 5
|
||||
if virtual_stream is not None:
|
||||
query["virtual_stream"] = virtual_stream
|
||||
if stream_version_lte is not None:
|
||||
query["stream_version_lte"] = stream_version_lte
|
||||
if base_module_br is not None:
|
||||
query["base_module_br"] = base_module_br
|
||||
modules = []
|
||||
|
||||
while True:
|
||||
res = requests_session.get(self.mbs_prod_url, params=query)
|
||||
if not res.ok:
|
||||
raise RuntimeError(self._generic_error % (query, res.status_code))
|
||||
res.raise_for_status()
|
||||
|
||||
data = res.json()
|
||||
modules_per_page = data["items"]
|
||||
@@ -84,23 +122,70 @@ class MBSResolver(KojiResolver):
|
||||
if not data["meta"]["next"]:
|
||||
break
|
||||
|
||||
if version is None and stream_version_lte is None:
|
||||
# Stop querying when we've gotten a different version
|
||||
if modules_per_page[-1]["version"] != modules[0]["version"]:
|
||||
break
|
||||
|
||||
query["page"] += 1
|
||||
|
||||
# Error handling
|
||||
if not modules:
|
||||
if strict:
|
||||
raise UnprocessableEntity("Failed to find module in MBS %r" % query)
|
||||
else:
|
||||
return modules
|
||||
|
||||
if version is None and "stream_version_lte" not in kwargs:
|
||||
if version is None and stream_version_lte is None:
|
||||
# Only return the latest version
|
||||
return [m for m in modules if m["version"] == modules[0]["version"]]
|
||||
results = [m for m in modules if m["version"] == modules[0]["version"]]
|
||||
else:
|
||||
results = modules
|
||||
|
||||
for m in results:
|
||||
# We often come back and query details again for the module builds we return
|
||||
# using the retrieved contexts, so prime the cache for a single-module queries
|
||||
self._get_module_by_nsvc.set(
|
||||
m, self, m["name"], m["stream"], m["version"], m["context"]
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def _get_modules(
|
||||
self, name, stream, version=None, context=None, states=None,
|
||||
base_module_br=None, virtual_stream=None, stream_version_lte=None,
|
||||
strict=False,
|
||||
|
||||
):
|
||||
"""Query and return modules from MBS with specific info
|
||||
|
||||
:param str name: module's name.
|
||||
:param str stream: module's stream.
|
||||
:kwarg str version: a string or int of the module's version. When None,
|
||||
latest version will be returned.
|
||||
:kwarg str context: module's context. Optional.
|
||||
:kwarg str states: states for modules. Defaults to ``["ready"]``.
|
||||
:kwarg str base_module_br: if set, restrict results to modules requiring
|
||||
this base_module nsvc
|
||||
:kwarg list virtual_stream: if set, limit to modules for the given virtual_stream
|
||||
:kwarg float stream_version_lte: If set, stream must be less than this version
|
||||
:kwarg bool strict: Normally this function returns None if no module can be
|
||||
found. If strict=True, then an UnprocessableEntity is raised.
|
||||
:return: final list of module_info which pass repoclosure
|
||||
:rtype: list[dict]
|
||||
"""
|
||||
modules = self._get_modules_with_cache(
|
||||
name, stream, version, context,
|
||||
states, base_module_br, virtual_stream, stream_version_lte
|
||||
)
|
||||
if not modules and strict:
|
||||
raise UnprocessableEntity("Failed to find module in MBS %s:%s:%s:%s" %
|
||||
(name, stream, version, context))
|
||||
else:
|
||||
return modules
|
||||
|
||||
def get_module(self, name, stream, version, context, states=None, strict=False):
|
||||
rv = self._get_modules(name, stream, version, context, states, strict)
|
||||
def get_module(self, name, stream, version, context, strict=False):
|
||||
if version and context:
|
||||
rv = self._get_module_by_nsvc(name, stream, version, context)
|
||||
if strict and rv is None:
|
||||
raise UnprocessableEntity("Failed to find module in MBS")
|
||||
|
||||
return rv
|
||||
|
||||
rv = self._get_modules(name, stream, version, context, strict=strict)
|
||||
if rv:
|
||||
return rv[0]
|
||||
|
||||
@@ -114,8 +199,7 @@ class MBSResolver(KojiResolver):
|
||||
query = {"page": 1, "per_page": 1, "short": True}
|
||||
query.update(kwargs)
|
||||
res = requests_session.get(self.mbs_prod_url, params=query)
|
||||
if not res.ok:
|
||||
raise RuntimeError(self._generic_error % (query, res.status_code))
|
||||
res.raise_for_status()
|
||||
|
||||
data = res.json()
|
||||
return data["meta"]["total"]
|
||||
@@ -138,23 +222,37 @@ class MBSResolver(KojiResolver):
|
||||
"virtual_stream": virtual_stream,
|
||||
}
|
||||
res = requests_session.get(self.mbs_prod_url, params=query)
|
||||
if not res.ok:
|
||||
raise RuntimeError(self._generic_error % (query, res.status_code))
|
||||
res.raise_for_status()
|
||||
|
||||
data = res.json()
|
||||
if data["items"]:
|
||||
return load_mmd(data["items"][0]["modulemd"])
|
||||
|
||||
def _modules_to_modulemds(self, modules, strict):
|
||||
if not modules:
|
||||
return []
|
||||
|
||||
mmds = []
|
||||
for module in modules:
|
||||
yaml = module["modulemd"]
|
||||
if not yaml:
|
||||
if strict:
|
||||
raise UnprocessableEntity(
|
||||
"Failed to find modulemd entry in MBS for %r" % module)
|
||||
else:
|
||||
log.warning("Failed to find modulemd entry in MBS for %r", module)
|
||||
continue
|
||||
|
||||
mmds.append(load_mmd(yaml))
|
||||
return mmds
|
||||
|
||||
def get_module_modulemds(
|
||||
self,
|
||||
name,
|
||||
stream,
|
||||
version=None,
|
||||
context=None,
|
||||
strict=False,
|
||||
stream_version_lte=False,
|
||||
virtual_streams=None,
|
||||
states=None,
|
||||
strict=False
|
||||
):
|
||||
"""
|
||||
Gets the module modulemds from the resolver.
|
||||
@@ -166,48 +264,15 @@ class MBSResolver(KojiResolver):
|
||||
be returned.
|
||||
:kwarg strict: Normally this function returns [] if no module can be
|
||||
found. If strict=True, then a UnprocessableEntity is raised.
|
||||
:kwarg stream_version_lte: If True and if the `stream` can be transformed to
|
||||
"stream version", the returned list will include all the modules with stream version
|
||||
less than or equal the stream version computed from `stream`.
|
||||
:kwarg virtual_streams: a list of the virtual streams to filter on. The filtering uses "or"
|
||||
logic. When falsy, no filtering occurs.
|
||||
:return: List of Modulemd metadata instances matching the query
|
||||
"""
|
||||
yaml = None
|
||||
|
||||
local_modules = models.ModuleBuild.local_modules(self.db_session, name, stream)
|
||||
if local_modules:
|
||||
return [m.mmd() for m in local_modules]
|
||||
|
||||
extra_args = {}
|
||||
if stream_version_lte and (
|
||||
len(str(models.ModuleBuild.get_stream_version(stream, right_pad=False))) >= 5
|
||||
):
|
||||
stream_version = models.ModuleBuild.get_stream_version(stream)
|
||||
extra_args["stream_version_lte"] = stream_version
|
||||
modules = self._get_modules(name, stream, version, context, strict=strict)
|
||||
|
||||
if virtual_streams:
|
||||
extra_args["virtual_stream"] = virtual_streams
|
||||
|
||||
modules = self._get_modules(name, stream, version, context, strict=strict, states=states,
|
||||
**extra_args)
|
||||
if not modules:
|
||||
return []
|
||||
|
||||
mmds = []
|
||||
for module in modules:
|
||||
if module:
|
||||
yaml = module["modulemd"]
|
||||
|
||||
if not yaml:
|
||||
if strict:
|
||||
raise UnprocessableEntity(
|
||||
"Failed to find modulemd entry in MBS for %r" % module)
|
||||
else:
|
||||
return None
|
||||
|
||||
mmds.append(load_mmd(yaml))
|
||||
return mmds
|
||||
return self._modules_to_modulemds(modules, strict)
|
||||
|
||||
def get_compatible_base_module_modulemds(
|
||||
self, base_module_mmd, stream_version_lte, virtual_streams, states
|
||||
@@ -225,18 +290,38 @@ class MBSResolver(KojiResolver):
|
||||
:param stream_version_lte: If True, the compatible streams are limited
|
||||
by the stream version computed from `stream`. If False, even the
|
||||
modules with higher stream version are returned.
|
||||
:param virtual_streams: List of virtual streams. If set, also modules
|
||||
with incompatible stream version are returned in case they share
|
||||
one of the virtual streams.
|
||||
:param virtual_streams: List of virtual streams.
|
||||
Modules are returned if they share one of the virtual streams.
|
||||
:param states: List of states the returned compatible modules should
|
||||
be in.
|
||||
:return list: List of Modulemd objects.
|
||||
"""
|
||||
if not virtual_streams:
|
||||
raise RuntimeError("Virtual stream list must not be empty")
|
||||
|
||||
name = base_module_mmd.get_module_name()
|
||||
stream = base_module_mmd.get_stream_name()
|
||||
return self.get_module_modulemds(
|
||||
name, stream, stream_version_lte=stream_version_lte, virtual_streams=virtual_streams,
|
||||
states=states)
|
||||
|
||||
extra_args = {}
|
||||
if stream_version_lte:
|
||||
stream = base_module_mmd.get_stream_name()
|
||||
stream_in_xyz_format = len(str(models.ModuleBuild.get_stream_version(
|
||||
stream, right_pad=False))) >= 5
|
||||
|
||||
if not stream_in_xyz_format:
|
||||
raise StreamNotXyz(
|
||||
"Cannot get compatible base modules, because stream of resolved "
|
||||
"base module %s:%s is not in x.y.z format." % (
|
||||
base_module_mmd.get_module_name(), stream
|
||||
))
|
||||
|
||||
stream_version = models.ModuleBuild.get_stream_version(stream)
|
||||
extra_args["stream_version_lte"] = stream_version
|
||||
|
||||
extra_args["virtual_stream"] = virtual_streams
|
||||
extra_args["states"] = _canonicalize_states(states)
|
||||
|
||||
modules = self._get_modules(name, None, **extra_args)
|
||||
return self._modules_to_modulemds(modules, False)
|
||||
|
||||
def get_buildrequired_modulemds(self, name, stream, base_module_mmd):
|
||||
"""
|
||||
@@ -269,7 +354,7 @@ class MBSResolver(KojiResolver):
|
||||
return ret
|
||||
else:
|
||||
modules = self._get_modules(
|
||||
name, stream, strict=False, base_module_br=base_module_mmd.get_nsvc())
|
||||
name, stream, base_module_br=base_module_mmd.get_nsvc())
|
||||
return [load_mmd(module["modulemd"]) for module in modules]
|
||||
|
||||
def resolve_profiles(self, mmd, keys):
|
||||
@@ -304,7 +389,7 @@ class MBSResolver(KojiResolver):
|
||||
continue
|
||||
|
||||
# Find the dep in the built modules in MBS
|
||||
modules = self._get_modules(
|
||||
module = self.get_module(
|
||||
module_name,
|
||||
module_info["stream"],
|
||||
module_info["version"],
|
||||
@@ -312,14 +397,13 @@ class MBSResolver(KojiResolver):
|
||||
strict=True,
|
||||
)
|
||||
|
||||
for module in modules:
|
||||
yaml = module["modulemd"]
|
||||
dep_mmd = load_mmd(yaml)
|
||||
# Take note of what rpms are in this dep's profile.
|
||||
for key in keys:
|
||||
profile = dep_mmd.get_profile(key)
|
||||
if profile:
|
||||
results[key] |= set(profile.get_rpms())
|
||||
yaml = module["modulemd"]
|
||||
dep_mmd = load_mmd(yaml)
|
||||
# Take note of what rpms are in this dep's profile.
|
||||
for key in keys:
|
||||
profile = dep_mmd.get_profile(key)
|
||||
if profile:
|
||||
results[key] |= set(profile.get_rpms())
|
||||
|
||||
# Return the union of all rpms in all profiles of the given keys.
|
||||
return results
|
||||
@@ -365,12 +449,15 @@ class MBSResolver(KojiResolver):
|
||||
else:
|
||||
queried_module = self.get_module(name, stream, version, context, strict=strict)
|
||||
yaml = queried_module["modulemd"]
|
||||
queried_mmd = load_mmd(yaml)
|
||||
if yaml:
|
||||
queried_mmd = load_mmd(yaml)
|
||||
else:
|
||||
queried_mmd = None
|
||||
|
||||
if not queried_mmd or "buildrequires" not in queried_mmd.get_xmd().get("mbs", {}):
|
||||
raise RuntimeError(
|
||||
'The module "{0!r}" did not contain its modulemd or did not have '
|
||||
"its xmd attribute filled out in MBS".format(queried_mmd)
|
||||
"its xmd attribute filled out in MBS".format(queried_module)
|
||||
)
|
||||
|
||||
buildrequires = queried_mmd.get_xmd()["mbs"]["buildrequires"]
|
||||
@@ -380,26 +467,23 @@ class MBSResolver(KojiResolver):
|
||||
self.db_session, name, details["stream"])
|
||||
if local_modules:
|
||||
for m in local_modules:
|
||||
# If the buildrequire is a meta-data only module with no Koji tag set, then just
|
||||
# skip it
|
||||
if m.koji_tag is None:
|
||||
continue
|
||||
module_tags[m.koji_tag] = m.mmd()
|
||||
continue
|
||||
|
||||
if "context" not in details:
|
||||
details["context"] = models.DEFAULT_MODULE_CONTEXT
|
||||
modules = self._get_modules(
|
||||
|
||||
m = self.get_module(
|
||||
name, details["stream"], details["version"], details["context"], strict=True)
|
||||
for m in modules:
|
||||
if m["koji_tag"] in module_tags:
|
||||
continue
|
||||
# If the buildrequire is a meta-data only module with no Koji tag set, then just
|
||||
# skip it
|
||||
if m["koji_tag"] is None:
|
||||
continue
|
||||
module_tags.setdefault(m["koji_tag"], [])
|
||||
module_tags[m["koji_tag"]].append(load_mmd(m["modulemd"]))
|
||||
|
||||
if m["koji_tag"] in module_tags:
|
||||
continue
|
||||
# If the buildrequire is a meta-data only module with no Koji tag set, then just
|
||||
# skip it
|
||||
if m["koji_tag"] is None:
|
||||
continue
|
||||
module_tags.setdefault(m["koji_tag"], [])
|
||||
module_tags[m["koji_tag"]].append(load_mmd(m["modulemd"]))
|
||||
|
||||
return module_tags
|
||||
|
||||
@@ -448,7 +532,6 @@ class MBSResolver(KojiResolver):
|
||||
|
||||
commit_hash = None
|
||||
version = None
|
||||
filtered_rpms = []
|
||||
module = self.get_module(
|
||||
module_name, module_stream, module_version, module_context, strict=True
|
||||
)
|
||||
@@ -457,20 +540,6 @@ class MBSResolver(KojiResolver):
|
||||
if mmd.get_xmd().get("mbs", {}).get("commit"):
|
||||
commit_hash = mmd.get_xmd()["mbs"]["commit"]
|
||||
|
||||
# Find out the particular NVR of filtered packages
|
||||
if "rpms" in module and mmd.get_rpm_filters():
|
||||
for rpm in module["rpms"]:
|
||||
nvr = kobo.rpmlib.parse_nvra(rpm)
|
||||
# If the package is not filtered, continue
|
||||
if not nvr["name"] in mmd.get_rpm_filters():
|
||||
continue
|
||||
|
||||
# If the nvr is already in filtered_rpms, continue
|
||||
nvr = kobo.rpmlib.make_nvr(nvr, force_epoch=True)
|
||||
if nvr in filtered_rpms:
|
||||
continue
|
||||
filtered_rpms.append(nvr)
|
||||
|
||||
if module.get("version"):
|
||||
version = module["version"]
|
||||
|
||||
@@ -481,11 +550,10 @@ class MBSResolver(KojiResolver):
|
||||
"version": str(version),
|
||||
"context": module["context"],
|
||||
"koji_tag": module["koji_tag"],
|
||||
"filtered_rpms": filtered_rpms,
|
||||
}
|
||||
else:
|
||||
raise RuntimeError(
|
||||
'The module "{0}" didn\'t contain either a commit hash or a'
|
||||
'The module "{0}" didn\'t contain both a commit hash and a'
|
||||
" version in MBS".format(module_name)
|
||||
)
|
||||
# If the module is a base module, then import it in the database so that entries in
|
||||
|
||||
@@ -78,7 +78,7 @@ class GenericResolver(six.with_metaclass(ABCMeta)):
|
||||
return False
|
||||
|
||||
@abstractmethod
|
||||
def get_module(self, name, stream, version, context, state="ready", strict=False):
|
||||
def get_module(self, name, stream, version, context, strict=False):
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@@ -83,7 +83,9 @@ def get_reusable_module(module):
|
||||
#
|
||||
# 1) The `conf.allow_only_compatible_base_modules` is False. This means that MBS should
|
||||
# not try to find any compatible base modules in its DB and simply use the buildrequired
|
||||
# base module as it is.
|
||||
# base module as it is. (*NOTE* This is different than the meaning of
|
||||
# allow_only_compatible_base_modules=False when looking up build requirements - where
|
||||
# it means to accept any module buildrequiring a base modul sharing a virtual stream.)
|
||||
# 2) The `conf.allow_only_compatible_base_modules` is True and DBResolver is used. This means
|
||||
# that MBS should try to find the compatible modules using its database.
|
||||
# The `get_base_module_mmds` finds out the list of compatible modules and returns mmds of
|
||||
|
||||
Reference in New Issue
Block a user