# -*- coding: utf-8 -*- # SPDX-License-Identifier: MIT from __future__ import absolute_import import sqlalchemy from sqlalchemy.orm import aliased from module_build_service.common import log, models from module_build_service.common.errors import StreamNotXyz, UnprocessableEntity from module_build_service.common.utils import load_mmd from module_build_service.resolver.base import GenericResolver class DBResolver(GenericResolver): """ Resolver using the MBS database """ backend = "db" def __init__(self, db_session, config): self.db_session = db_session self.config = config def get_module( self, name, stream, version, context, strict=False ): mb = models.ModuleBuild.get_build_from_nsvc( self.db_session, name, stream, version, context, state=models.BUILD_STATES["ready"]) if mb: return mb.extended_json(self.db_session) if strict: raise UnprocessableEntity( "Cannot find any module builds for %s:%s" % (name, stream)) def get_module_count(self, **kwargs): """ Determine the number of modules that match the provided filter. :return: the number of modules that match the provided filter :rtype: int """ return models.ModuleBuild.get_module_count(self.db_session, **kwargs) def get_latest_with_virtual_stream(self, name, virtual_stream): """ Get the latest module with the input virtual stream based on the stream version and version. :param str name: the module name to search for :param str virtual_stream: the module virtual stream to search for :return: the module's modulemd or None :rtype: Modulemd.ModuleStream or None """ query = self.db_session.query(models.ModuleBuild).filter_by(name=name) query = models.ModuleBuild._add_virtual_streams_filter( self.db_session, query, [virtual_stream]) # Cast the version as an integer so that we get proper ordering module = query.order_by( models.ModuleBuild.stream_version.desc(), sqlalchemy.cast(models.ModuleBuild.version, sqlalchemy.BigInteger).desc(), ).first() if module: return load_mmd(module.modulemd) def get_module_modulemds(self, name, stream, version=None, context=None, strict=False): """ Gets the module modulemds from the resolver. :param name: a string of the module's name :param stream: a string of the module's stream :param version: a string or int of the module's version. When None, latest version will be returned. :param context: a string of the module's context. When None, all contexts will be returned. :kwarg strict: Normally this function returns [] if no module can be found. If strict=True, then a UnprocessableEntity is raised. :return: List of Modulemd metadata instances matching the query """ if version and context: mmd = self.get_module(name, stream, version, context, strict=strict) if mmd is None: return return [load_mmd(mmd["modulemd"])] if not version and not context: builds = models.ModuleBuild.get_last_builds_in_stream(self.db_session, name, stream) else: raise NotImplementedError( "This combination of name/stream/version/context is not implemented") if not builds and strict: raise UnprocessableEntity( "Cannot find any module builds for %s:%s" % (name, stream)) return [build.mmd() for build in builds] def get_compatible_base_module_modulemds( self, base_module_mmd, stream_version_lte, virtual_streams, states ): """ Returns the Modulemd metadata of base modules compatible with base module defined by `name` and `stream`. The compatibility is found out using the stream version in case the stream is in "x.y.z" format and is limited to single major version of stream version. If `virtual_streams` are defined, the compatibility is also extended to all base module streams which share the same virtual stream. :param base_module_mmd: Modulemd medatada defining the input base module. :param stream_version_lte: If True, the compatible streams are limited by the stream version computed from `stream`. If False, even the modules with higher stream version are returned. :param virtual_streams: List of virtual streams. Modules are returned if they share one of the virtual streams. :param states: List of states the returned compatible modules should be in. :return list: List of Modulemd objects. """ if not virtual_streams: raise RuntimeError("Virtual stream list must not be empty") name = base_module_mmd.get_module_name() stream = base_module_mmd.get_stream_name() builds = [] stream_version = None if stream_version_lte: stream_in_xyz_format = len(str(models.ModuleBuild.get_stream_version( stream, right_pad=False))) >= 5 if not stream_in_xyz_format: raise StreamNotXyz( "Cannot get compatible base modules, because stream of resolved " "base module %s:%s is not in x.y.z format." % ( base_module_mmd.get_module_name(), stream )) stream_version = models.ModuleBuild.get_stream_version(stream) builds = models.ModuleBuild.get_last_builds_in_stream_version_lte( self.db_session, name, stream_version, virtual_streams, states) return [build.mmd() for build in builds] def get_buildrequired_modulemds(self, name, stream, base_module_mmd): """ Returns modulemd metadata of all module builds with `name` and `stream` buildrequiring base module defined by `base_module_mmd` NSVC. :param str name: Name of module to return. :param str stream: Stream of module to return. :param Modulemd base_module_mmd: NSVC of base module which must be buildrequired by returned modules. :rtype: list :return: List of modulemd metadata. """ log.debug("Looking for %s:%s buildrequiring %s", name, stream, base_module_mmd.get_nsvc()) query = self.db_session.query(models.ModuleBuild) query = query.filter_by(name=name, stream=stream, state=models.BUILD_STATES["ready"]) module_br_alias = aliased(models.ModuleBuild, name="module_br") # Shorten this table name for clarity in the query below mb_to_br = models.module_builds_to_module_buildrequires # The following joins get added: # JOIN module_builds_to_module_buildrequires # ON module_builds_to_module_buildrequires.module_id = module_builds.id # JOIN module_builds AS module_br # ON module_builds_to_module_buildrequires.module_buildrequire_id = module_br.id query = query.join(mb_to_br, mb_to_br.c.module_id == models.ModuleBuild.id).join( module_br_alias, mb_to_br.c.module_buildrequire_id == module_br_alias.id) # Get only modules buildrequiring particular base_module_mmd n, s, v, c = base_module_mmd.get_nsvc().split(":") query = query.filter( module_br_alias.name == n, module_br_alias.stream == s, module_br_alias.version == v, module_br_alias.context == c, ) query = query.order_by( sqlalchemy.cast(models.ModuleBuild.version, sqlalchemy.BigInteger).desc()) all_builds = query.all() # The `all_builds` list contains builds sorted by "build.version". We need only # the builds with latest version, but in all contexts. builds = [] latest_version = None for build in all_builds: if latest_version is None: latest_version = build.version if latest_version != build.version: break builds.append(build) mmds = [build.mmd() for build in builds] nsvcs = [ mmd.get_nsvc() for mmd in mmds ] log.debug("Found: %r", nsvcs) return mmds def resolve_profiles(self, mmd, keys): """ Returns a dictionary with keys set according the `keys` parameters and values set to the union of all components defined in all installation profiles matching the key in all buildrequires. If there are some modules loaded by load_local_builds(...), these local modules will be considered when returning the profiles. :param mmd: Modulemd.ModuleStream instance representing the module :param keys: list of modulemd installation profiles to include in the result :return: a dictionary """ results = {} for key in keys: results[key] = set() for module_name, module_info in mmd.get_xmd()["mbs"]["buildrequires"].items(): local_modules = models.ModuleBuild.local_modules( self.db_session, module_name, module_info["stream"]) if local_modules: local_module = local_modules[0] log.info("Using local module {0!r} to resolve profiles.".format(local_module)) dep_mmd = local_module.mmd() for key in keys: profile = dep_mmd.get_profile(key) if profile: results[key] |= set(profile.get_rpms()) continue build = models.ModuleBuild.get_build_from_nsvc( self.db_session, module_name, module_info["stream"], module_info["version"], module_info["context"], state=models.BUILD_STATES["ready"], ) if not build: raise UnprocessableEntity( "The module {}:{}:{}:{} was not found".format( module_name, module_info["stream"], module_info["version"], module_info["context"], ) ) dep_mmd = build.mmd() # Take note of what rpms are in this dep's profile for key in keys: profile = dep_mmd.get_profile(key) if profile: results[key] |= set(profile.get_rpms()) # Return the union of all rpms in all profiles of the given keys return results def get_module_build_dependencies( self, name=None, stream=None, version=None, context=None, mmd=None, strict=False ): """ Returns a dictionary of koji_tag:[mmd, ...] of all the dependencies of input module. Although it is expected that single Koji tag always contain just single module build, it does not have to be a true for Offline local builds which use the local repository identifier as `koji_tag`. :kwarg name: a string of a module's name (required if mmd is not set) :kwarg stream: a string of a module's stream (required if mmd is not set) :kwarg version: a string of a module's version (required if mmd is not set) :kwarg context: a string of a module's context (required if mmd is not set) :kwarg mmd: Modulemd.ModuleStream object. If this is set, the mmd will be used instead of querying the DB with the name, stream, version, and context. :kwarg strict: Normally this function returns None if no module can be found. If strict=True, then an UnprocessableEntity is raised. :return: a dictionary """ if mmd: log.debug("get_module_build_dependencies(mmd={0!r} strict={1!r})".format(mmd, strict)) elif any(x is None for x in [name, stream, version, context]): raise RuntimeError("The name, stream, version, and/or context weren't specified") else: version = str(version) log.debug( "get_module_build_dependencies({0}, strict={1!r})".format( ", ".join([name, stream, str(version), context]), strict) ) module_build = None module_tags = {} if mmd: queried_mmd = mmd nsvc = ":".join([ mmd.get_module_name(), mmd.get_stream_name(), str(mmd.get_version()), mmd.get_context() or models.DEFAULT_MODULE_CONTEXT, ]) else: module_build = models.ModuleBuild.get_build_from_nsvc( self.db_session, name, stream, version, context) if not module_build: raise UnprocessableEntity( "The module {} was not found".format( ":".join([name, stream, version, context])) ) queried_mmd = module_build.mmd() nsvc = ":".join([name, stream, version, context]) xmd_mbs = queried_mmd.get_xmd().get("mbs", {}) if "buildrequires" not in xmd_mbs: raise RuntimeError( "The module {} did not contain its modulemd or did not have " "its xmd attribute filled out in MBS".format(nsvc) ) buildrequires = xmd_mbs["buildrequires"] side_tag = xmd_mbs.get("side_tag") side_tag_brs_found = [] for br_name, details in buildrequires.items(): build = models.ModuleBuild.get_build_from_nsvc( self.db_session, br_name, details["stream"], details["version"], details["context"], state=models.BUILD_STATES["ready"], ) if not build: raise RuntimeError( "Buildrequired module %s %r does not exist in MBS db" % (br_name, details)) koji_tag = build.koji_tag if side_tag and br_name in self.config.base_module_names: # see if base module has a side tag configuration side_tag_format = build.mmd().get_xmd().get("mbs", {}).get("koji_side_tag_format") if side_tag_format: koji_tag = side_tag_format.format(side_tag=side_tag) side_tag_brs_found.append(side_tag_format) log.info("Using side tag for base module %s: %s", br_name, koji_tag) else: log.warning("Side tag requested, but base module %s lacks koji_side_tag_format" " value", br_name) # If the buildrequire is a meta-data only module with no Koji tag set, then just # skip it if koji_tag is None: continue module_tags.setdefault(koji_tag, []) module_tags[koji_tag].append(build.mmd()) if side_tag and not side_tag_brs_found: msg = "None of the buildrequired base modules are configured for side tags." if module_build: module_build.log_message(self.db_session, msg) raise RuntimeError(msg) return module_tags def resolve_requires(self, requires): """ Resolves the requires list of N:S or N:S:V:C to a dictionary with keys as the module name and the values as a dictionary with keys of ref, stream, version. If there are some modules loaded by utils.load_local_builds(...), these local modules will be considered when resolving the requires. A RuntimeError is raised on DB lookup errors. :param requires: a list of N:S or N:S:V:C strings :return: a dictionary """ new_requires = {} for nsvc in requires: nsvc_splitted = nsvc.split(":") if len(nsvc_splitted) == 2: module_name, module_stream = nsvc_splitted module_version = None module_context = None elif len(nsvc_splitted) == 4: module_name, module_stream, module_version, module_context = nsvc_splitted else: raise ValueError( "Only N:S or N:S:V:C is accepted by resolve_requires, got %s" % nsvc) local_modules = models.ModuleBuild.local_modules( self.db_session, module_name, module_stream) if local_modules: local_build = local_modules[0] new_requires[module_name] = { "ref": None, "stream": local_build.stream, "version": local_build.version, "context": local_build.context, "koji_tag": local_build.koji_tag, } continue if module_version is None or module_context is None: build = models.ModuleBuild.get_last_build_in_stream( self.db_session, module_name, module_stream ) else: build = models.ModuleBuild.get_build_from_nsvc( self.db_session, module_name, module_stream, module_version, module_context ) if not build: raise UnprocessableEntity("The module {} was not found".format(nsvc)) for sibling_id in build.siblings(self.db_session): sibling_build = models.ModuleBuild.get_by_id(self.db_session, sibling_id) if sibling_build.state not in ( models.BUILD_STATES["ready"], models.BUILD_STATES["failed"], models.BUILD_STATES["garbage"] ): raise UnprocessableEntity('Buildrequire {}-{}-{} is in "{}" state'.format( sibling_build.name, sibling_build.stream, sibling_build.version, models.INVERSE_BUILD_STATES[sibling_build.state] )) commit_hash = None mmd = build.mmd() mbs_xmd = mmd.get_xmd().get("mbs", {}) if mbs_xmd.get("commit"): commit_hash = mbs_xmd["commit"] else: raise RuntimeError( 'The module "{0}" didn\'t contain a commit hash in its xmd'.format( module_name) ) if not mbs_xmd.get("mse"): raise RuntimeError( 'The module "{}" is not built using Module Stream Expansion. ' "Please rebuild this module first".format(nsvc) ) new_requires[module_name] = { "ref": commit_hash, "stream": module_stream, "version": build.version, "context": build.context, "koji_tag": build.koji_tag, } return new_requires def get_modulemd_by_koji_tag(self, tag): module = models.ModuleBuild.get_build_by_koji_tag(self.db_session, tag) return module.mmd() if module else None