mirror of
https://pagure.io/fm-orchestrator.git
synced 2026-04-09 13:49:24 +08:00
6fd3a2586d
Modules that might have been added to a Mock build with --add-local-build were only checked in one place, but there were two other places where data was loaded directly from the PDC.
382 lines
16 KiB
Python
382 lines
16 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
|
|
# Copyright (c) 2018 Red Hat, Inc.
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
# of this software and associated documentation files (the "Software"), to deal
|
|
# in the Software without restriction, including without limitation the rights
|
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
# copies of the Software, and to permit persons to whom the Software is
|
|
# furnished to do so, subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in all
|
|
# copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
# SOFTWARE.
|
|
#
|
|
# Written by Lubos Kocman <lkocman@redhat.com>
|
|
# Filip Valder <fvalder@redhat.com>
|
|
|
|
"""PDC handler functions."""
|
|
|
|
import pdc_client
|
|
from module_build_service import db
|
|
from module_build_service import models
|
|
from module_build_service.errors import UnprocessableEntity
|
|
from module_build_service.resolver.base import GenericResolver
|
|
|
|
import inspect
|
|
import logging
|
|
import kobo.rpmlib
|
|
log = logging.getLogger()
|
|
|
|
|
|
class PDCResolver(GenericResolver):
|
|
|
|
backend = "pdc"
|
|
|
|
def __init__(self, config):
|
|
self.config = config
|
|
self.session = self._get_client_session(self.config)
|
|
|
|
def _get_client_session(self, config):
|
|
"""
|
|
:param config: instance of module_build_service.config.Config
|
|
:return pdc_client.PDCClient instance
|
|
"""
|
|
if 'ssl_verify' in inspect.getargspec(pdc_client.PDCClient.__init__).args:
|
|
# New API
|
|
return pdc_client.PDCClient(
|
|
server=self.config.pdc_url,
|
|
develop=self.config.pdc_develop,
|
|
ssl_verify=not self.config.pdc_insecure,
|
|
)
|
|
else:
|
|
# Old API
|
|
return pdc_client.PDCClient(
|
|
server=self.config.pdc_url,
|
|
develop=self.config.pdc_develop,
|
|
insecure=self.config.pdc_insecure,
|
|
)
|
|
|
|
def _query_from_nsvc(self, name, stream, version=None, context=None, active=None):
|
|
"""
|
|
Generates dict with PDC query.
|
|
|
|
:param str name: Name of the module to query.
|
|
:param str stream: Stream of the module to query.
|
|
:param str version/int: Version of the module to query.
|
|
:param str context: Context of the module to query.
|
|
:param bool active: Include "active" in a query.
|
|
"""
|
|
query = {
|
|
"variant_id": name,
|
|
"variant_version": stream,
|
|
}
|
|
if active is not None:
|
|
query["active"] = active
|
|
if version is not None:
|
|
query["variant_release"] = str(version)
|
|
if context is not None:
|
|
query["variant_context"] = context
|
|
return query
|
|
|
|
def _get_modules(self, name, stream, version=None, context=None, active=None, strict=False):
|
|
"""
|
|
:param module_info: pdc variant_dict, str, mmd or module dict
|
|
:param strict: Normally this function returns None if no module can be
|
|
found. If strict=True, then an UnprocessableEntity is raised.
|
|
|
|
:return final list of module_info which pass repoclosure
|
|
"""
|
|
query = self._query_from_nsvc(name, stream, version, context, active)
|
|
|
|
# TODO: So far sorting on Fedora prod PDC instance is broken and it sorts
|
|
# only by variant_uid by default. Once the sorting is fixed, we can start
|
|
# using '-variant_release' ordering and just get the first variant from
|
|
# there. But in the meantime, we have to get the first variant with
|
|
# page_size set to 1 to find out how many variants (pages) are there in
|
|
# results set and jump to last one in another query. The last one is always
|
|
# the latest one (the one with the highest version).
|
|
try:
|
|
retval = self.session['unreleasedvariants']._(page_size=1, **query)
|
|
except Exception as ex:
|
|
log.debug("error during PDC lookup: %r" % ex)
|
|
raise RuntimeError("Error during PDC lookup for module %s" % name)
|
|
|
|
# Error handling
|
|
if not retval or len(retval["results"]) == 0:
|
|
if strict:
|
|
raise UnprocessableEntity("Failed to find module in PDC %r" % query)
|
|
else:
|
|
return None
|
|
|
|
# Get all the contexts of given module in case there is just NS or NSV input.
|
|
if not version or not context:
|
|
# If the version is not set, we have to jump to last page to get the latest
|
|
# version in PDC.
|
|
if not version:
|
|
query['page'] = retval['count']
|
|
retval = self.session['unreleasedvariants']._(page_size=1, **query)
|
|
del query['page']
|
|
query["variant_release"] = retval["results"][0]["variant_release"]
|
|
retval = self.session['unreleasedvariants']._(page_size=-1, **query)
|
|
return retval
|
|
|
|
results = retval["results"]
|
|
# Error handling
|
|
if len(results) == 0:
|
|
if strict:
|
|
raise UnprocessableEntity("Failed to find module in PDC %r" % query)
|
|
else:
|
|
return None
|
|
return results
|
|
|
|
def _get_module(self, name, stream, version=None, context=None, active=None, strict=False):
|
|
return self._get_modules(name, stream, version, context, active, strict)[0]
|
|
|
|
def get_module_modulemds(self, name, stream, version=None, context=None, strict=False):
|
|
"""
|
|
Gets the module modulemds from the resolver.
|
|
:param name: a string of the module's name
|
|
:param stream: a string of the module's stream
|
|
:param version: a string or int of the module's version. When None, latest version will
|
|
be returned.
|
|
:param context: a string of the module's context. When None, all contexts will
|
|
be returned.
|
|
:kwarg strict: Normally this function returns [] if no module can be
|
|
found. If strict=True, then a UnprocessableEntity is raised.
|
|
:return: List of Modulemd metadata instances matching the query
|
|
"""
|
|
yaml = None
|
|
|
|
local_modules = models.ModuleBuild.local_modules(db.session, name, stream)
|
|
if local_modules:
|
|
return [m.mmd() for m in local_modules]
|
|
|
|
modules = self._get_modules(name, stream, version, context, active=True, strict=strict)
|
|
if not modules:
|
|
return []
|
|
|
|
mmds = []
|
|
for module in modules:
|
|
if module:
|
|
yaml = module['modulemd']
|
|
|
|
if not yaml:
|
|
if strict:
|
|
raise UnprocessableEntity(
|
|
"Failed to find modulemd entry in PDC for %r" % module)
|
|
else:
|
|
return None
|
|
|
|
mmds.append(self.extract_modulemd(yaml, strict=strict))
|
|
return mmds
|
|
|
|
def resolve_profiles(self, mmd, keys):
|
|
"""
|
|
:param mmd: ModuleMetadata instance of module
|
|
:param keys: list of modulemd installation profiles to include in
|
|
the result.
|
|
:return: Dictionary with keys set according to `keys` param and values
|
|
set to union of all components defined in all installation
|
|
profiles matching the key using the buildrequires.
|
|
|
|
If there are some modules loaded by utils.load_local_builds(...), these
|
|
local modules will be considered when returning the profiles.
|
|
|
|
https://pagure.io/fm-orchestrator/issue/181
|
|
"""
|
|
|
|
results = {}
|
|
for key in keys:
|
|
results[key] = set()
|
|
for module_name, module_info in mmd.get_xmd()['mbs']['buildrequires'].items():
|
|
local_modules = models.ModuleBuild.local_modules(
|
|
db.session, module_name, module_info['stream'])
|
|
if local_modules:
|
|
local_module = local_modules[0]
|
|
log.info("Using local module %r to resolve profiles.",
|
|
local_module)
|
|
dep_mmd = local_module.mmd()
|
|
for key in keys:
|
|
if key in dep_mmd.get_profiles().keys():
|
|
results[key] |= set(dep_mmd.get_profiles()[key].get_rpms().get())
|
|
continue
|
|
|
|
# Find the dep in the built modules in PDC
|
|
modules = self._get_modules(
|
|
module_name, module_info['stream'], module_info['version'],
|
|
module_info['context'], strict=True)
|
|
|
|
for module in modules:
|
|
yaml = module['modulemd']
|
|
dep_mmd = self.extract_modulemd(yaml)
|
|
# Take note of what rpms are in this dep's profile.
|
|
for key in keys:
|
|
if key in dep_mmd.get_profiles().keys():
|
|
results[key] |= set(dep_mmd.get_profiles()[key].get_rpms().get())
|
|
|
|
# Return the union of all rpms in all profiles of the given keys.
|
|
return results
|
|
|
|
def get_module_build_dependencies(self, name=None, stream=None, version=None, context=None,
|
|
mmd=None, strict=False):
|
|
"""
|
|
:param name: a module's name (required if mmd is not set)
|
|
:param stream: a module's stream (required if mmd is not set)
|
|
:param version: a module's version (required if mmd is not set)
|
|
:param context: a module's context (required if mmd is not set)
|
|
:param mmd: uses the mmd instead of the name, stream, version to query PDC
|
|
:param strict: Normally this function returns None if no module can be
|
|
found. If strict=True, then an UnprocessableEntity is raised.
|
|
:return dict with koji_tag as a key and ModuleMetadata object as value.
|
|
"""
|
|
if mmd:
|
|
log.debug("get_module_build_dependencies(mmd=%r strict=%r)" % (mmd, strict))
|
|
elif any(x is None for x in [name, stream, version, context]):
|
|
raise RuntimeError('The name, stream, version, and/or context weren\'t specified')
|
|
else:
|
|
version = str(version)
|
|
log.debug("get_module_build_dependencies(%s, strict=%r)"
|
|
% (', '.join([name, stream, str(version), context]), strict))
|
|
|
|
# This is the set we're going to build up and return.
|
|
module_tags = {}
|
|
|
|
if mmd:
|
|
queried_mmd = mmd
|
|
else:
|
|
queried_module = self._get_module(
|
|
name, stream, version, context, strict=strict)
|
|
yaml = queried_module['modulemd']
|
|
queried_mmd = self.extract_modulemd(yaml, strict=strict)
|
|
|
|
if (not queried_mmd or not queried_mmd.get_xmd().get('mbs') or
|
|
'buildrequires' not in queried_mmd.get_xmd()['mbs'].keys()):
|
|
raise RuntimeError(
|
|
'The module "{0!r}" did not contain its modulemd or did not have '
|
|
'its xmd attribute filled out in PDC'.format(queried_mmd))
|
|
|
|
buildrequires = queried_mmd.get_xmd()['mbs']['buildrequires']
|
|
# Queue up the next tier of deps that we should look at..
|
|
for name, details in buildrequires.items():
|
|
local_modules = models.ModuleBuild.local_modules(
|
|
db.session, name, details['stream'])
|
|
if local_modules:
|
|
for m in local_modules:
|
|
module_tags[m.koji_tag] = m.mmd()
|
|
continue
|
|
|
|
modules = self._get_modules(
|
|
name, details['stream'], details['version'],
|
|
details['context'], active=True, strict=True)
|
|
for m in modules:
|
|
if m["koji_tag"] in module_tags:
|
|
continue
|
|
module_tags[m["koji_tag"]] = self.extract_modulemd(m["modulemd"])
|
|
|
|
return module_tags
|
|
|
|
def resolve_requires(self, requires):
|
|
"""
|
|
Takes `requires` dict with module_name as key and module_stream as value.
|
|
Resolves the stream to particular latest version of a module and returns
|
|
new dict in following format:
|
|
|
|
{
|
|
"module_name": {
|
|
"ref": module_commit_hash,
|
|
"stream": original_module_stream,
|
|
"version": module_version,
|
|
"filtered_rpms": ["nvr", ...]
|
|
},
|
|
...
|
|
}
|
|
|
|
If there are some modules loaded by utils.load_local_builds(...), these
|
|
local modules will be considered when resolving the requires.
|
|
|
|
Raises RuntimeError on PDC lookup error.
|
|
"""
|
|
new_requires = {}
|
|
for nsvc in requires:
|
|
nsvc_splitted = nsvc.split(":")
|
|
if len(nsvc_splitted) == 2:
|
|
module_name, module_stream = nsvc_splitted
|
|
module_version = None
|
|
module_context = None
|
|
elif len(nsvc_splitted) == 4:
|
|
module_name, module_stream, module_version, module_context = nsvc_splitted
|
|
else:
|
|
raise ValueError(
|
|
"Only N:S or N:S:V:C is accepted by resolve_requires, got %s" % nsvc)
|
|
# Try to find out module dependency in the local module builds
|
|
# added by utils.load_local_builds(...).
|
|
local_modules = models.ModuleBuild.local_modules(
|
|
db.session, module_name, module_stream)
|
|
if local_modules:
|
|
local_build = local_modules[0]
|
|
new_requires[module_name] = {
|
|
# The commit ID isn't currently saved in modules.yaml
|
|
'ref': None,
|
|
'stream': local_build.stream,
|
|
'version': local_build.version,
|
|
'context': local_build.context,
|
|
# No need to set filtered_rpms for local builds, because MBS
|
|
# filters the RPMs automatically when the module build is
|
|
# done.
|
|
'filtered_rpms': []
|
|
}
|
|
continue
|
|
|
|
commit_hash = None
|
|
version = None
|
|
filtered_rpms = []
|
|
module = self._get_module(
|
|
module_name, module_stream, module_version,
|
|
module_context, active=True, strict=True)
|
|
if module.get('modulemd'):
|
|
mmd = self.extract_modulemd(module['modulemd'])
|
|
if mmd.get_xmd().get('mbs') and 'commit' in mmd.get_xmd()['mbs'].keys():
|
|
commit_hash = mmd.get_xmd()['mbs']['commit']
|
|
|
|
# Find out the particular NVR of filtered packages
|
|
if "rpms" in module and mmd.get_rpm_filter().get():
|
|
for rpm in module["rpms"]:
|
|
nvr = kobo.rpmlib.parse_nvra(rpm)
|
|
# If the package is not filtered, continue
|
|
if not nvr["name"] in mmd.get_rpm_filter().get():
|
|
continue
|
|
|
|
# If the nvr is already in filtered_rpms, continue
|
|
nvr = kobo.rpmlib.make_nvr(nvr, force_epoch=True)
|
|
if nvr in filtered_rpms:
|
|
continue
|
|
filtered_rpms.append(nvr)
|
|
|
|
if module.get('variant_release'):
|
|
version = module['variant_release']
|
|
|
|
if version and commit_hash:
|
|
new_requires[module_name] = {
|
|
'ref': commit_hash,
|
|
'stream': module_stream,
|
|
'version': str(version),
|
|
'context': module["variant_context"],
|
|
'filtered_rpms': filtered_rpms,
|
|
}
|
|
else:
|
|
raise RuntimeError(
|
|
'The module "{0}" didn\'t contain either a commit hash or a'
|
|
' version in PDC'.format(module_name))
|
|
|
|
return new_requires
|