diff --git a/docs/VIRTUAL_MODULES.rst b/docs/VIRTUAL_MODULES.rst index 4432d94f..6fabd027 100644 --- a/docs/VIRTUAL_MODULES.rst +++ b/docs/VIRTUAL_MODULES.rst @@ -66,6 +66,9 @@ Custom fields in xmd: - ``koji_tag`` - this defines the Koji tag with the RPMs that are part of this module. For base modules this will likely be a tag representing a buildroot. If this is a metadata-only module, then this can be left unset. +- ``koji_tag_with_modules`` - this defines the Koji tag with the module builds. These modules are + later used to fulfill the build requirements of modules built on against this module. This + option is used only when ``KojiResolver`` is enabled on the MBS server. - ``virtual_streams`` - the list of streams which groups multiple modules together. For more information on this field, see the ``Virtual Streams`` section below. - ``disttag_marking`` - if this module is a base module, then MBS will use the stream of the base diff --git a/module_build_service/resolver/DBResolver.py b/module_build_service/resolver/DBResolver.py index 2dbb9fba..fa362575 100644 --- a/module_build_service/resolver/DBResolver.py +++ b/module_build_service/resolver/DBResolver.py @@ -155,19 +155,19 @@ class DBResolver(GenericResolver): return [build.mmd() for build in builds] - def get_buildrequired_modulemds(self, name, stream, base_module_nsvc): + def get_buildrequired_modulemds(self, name, stream, base_module_mmd): """ Returns modulemd metadata of all module builds with `name` and `stream` buildrequiring - base module defined by `base_module_nsvc` NSVC. + base module defined by `base_module_mmd` NSVC. :param str name: Name of module to return. :param str stream: Stream of module to return. - :param str base_module_nsvc: NSVC of base module which must be buildrequired by returned + :param Modulemd base_module_mmd: NSVC of base module which must be buildrequired by returned modules. :rtype: list :return: List of modulemd metadata. """ - log.debug("Looking for %s:%s buildrequiring %s", name, stream, base_module_nsvc) + log.debug("Looking for %s:%s buildrequiring %s", name, stream, base_module_mmd.get_nsvc()) query = self.db_session.query(models.ModuleBuild) query = query.filter_by(name=name, stream=stream, state=models.BUILD_STATES["ready"]) @@ -182,8 +182,8 @@ class DBResolver(GenericResolver): query = query.join(mb_to_br, mb_to_br.c.module_id == models.ModuleBuild.id).join( module_br_alias, mb_to_br.c.module_buildrequire_id == module_br_alias.id) - # Get only modules buildrequiring particular base_module_nsvc - n, s, v, c = base_module_nsvc.split(":") + # Get only modules buildrequiring particular base_module_mmd + n, s, v, c = base_module_mmd.get_nsvc().split(":") query = query.filter( module_br_alias.name == n, module_br_alias.stream == s, diff --git a/module_build_service/resolver/KojiResolver.py b/module_build_service/resolver/KojiResolver.py new file mode 100644 index 00000000..39d282a8 --- /dev/null +++ b/module_build_service/resolver/KojiResolver.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza + +from itertools import groupby + +from module_build_service.resolver.DBResolver import DBResolver +from module_build_service import conf, models, log + + +class KojiResolver(DBResolver): + """ + Resolver using Koji server running in infrastructure. + """ + + backend = "koji" + + def _filter_inherited(self, koji_session, module_builds, top_tag, event): + """ + Look at the tag inheritance and keep builds only from the topmost tag. + + For example, we have "foo:bar:1" and "foo:bar:2" builds. We also have "foo-tag" which + inherits "foo-parent-tag". The "foo:bar:1" is tagged in the "foo-tag". The "foo:bar:2" + is tagged in the "foo-parent-tag". + + In this case, this function filters out the foo:bar:2, because "foo:bar:1" is tagged + lower in the inheritance tree in the "foo-tag". + + For normal RPMs, using latest=True for listTagged() call, Koji would automatically do + this, but it does not understand streams, so we have to reimplement it here. + + :param KojiSession koji_session: Koji session. + :param list module_builds: List of builds as returned by KojiSession.listTagged method. + :param str top_tag: The top Koji tag. + :param dict event: Koji event defining the time at which the `module_builds` have been + fetched. + :return list: Filtered list of builds. + """ + inheritance = [ + tag["name"] for tag in koji_session.getFullInheritance(top_tag, event=event["id"]) + ] + + def keyfunc(mb): + return (mb["name"], mb["version"]) + + result = [] + + # Group modules by Name-Stream + for _, builds in groupby(sorted(module_builds, key=keyfunc), keyfunc): + builds = list(builds) + # For each N-S combination find out which tags it's in + available_in = set(build["tag_name"] for build in builds) + + # And find out which is the topmost tag + for tag in [top_tag] + inheritance: + if tag in available_in: + break + + # And keep only builds from that topmost tag + result.extend(build for build in builds if build["tag_name"] == tag) + + return result + + def get_buildrequired_modulemds(self, name, stream, base_module_mmd): + """ + Returns modulemd metadata of all module builds with `name` and `stream` which are tagged + in the Koji tag defined in `base_module_mmd`. + + :param str name: Name of module to return. + :param str stream: Stream of module to return. + :param Modulemd base_module_mmd: Base module metadata. + :return list: List of modulemd metadata. + """ + # Get the `koji_tag_with_modules`. If the `koji_tag_with_modules` is not configured for + # the base module, fallback to DBResolver. + tag = base_module_mmd.get_xmd().get("mbs", {}).get("koji_tag_with_modules") + if not tag: + log.info( + "The %s does not define 'koji_tag_with_modules'. Falling back to DBResolver." % ( + base_module_mmd.get_nsvc())) + return DBResolver.get_buildrequired_modulemds(self, name, stream, base_module_mmd) + + # Create KojiSession. We need to import here because of circular dependencies. + from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder + koji_session = KojiModuleBuilder.get_session(conf, login=False) + event = koji_session.getLastEvent() + + # List all the modular builds in the modular Koji tag. + # We cannot use latest=True here, because we need to get all the + # available streams of all modules. The stream is represented as + # "version" in Koji build and with latest=True, Koji would return + # only builds with the highest version. + # We also cannot ask for particular `stream`, because Koji does not support that. + module_builds = koji_session.listTagged( + tag, inherit=True, type="module", package=name, event=event["id"]) + + # Filter out different streams + normalized_stream = stream.replace("-", "_") + module_builds = [b for b in module_builds if b["version"] == normalized_stream] + + # Filter out builds inherited from non-top tag + module_builds = self._filter_inherited(koji_session, module_builds, tag, event) + + # Find the latest builds of all modules. This does the following: + # - Sorts the module_builds descending by Koji NVR (which maps to NSV + # for modules). Split release into modular version and context, and + # treat version as numeric. + # - Groups the sorted module_builds by NV (NS in modular world). + # In each resulting `ns_group`, the first item is actually build + # with the latest version (because the list is still sorted by NVR). + # - Groups the `ns_group` again by "release" ("version" in modular + # world) to just get all the "contexts" of the given NSV. This is + # stored in `nsv_builds`. + # - The `nsv_builds` contains the builds representing all the contexts + # of the latest version for give name-stream, so add them to + # `latest_builds`. + def _key(build): + ver, ctx = build["release"].split(".", 1) + return build["name"], build["version"], int(ver), ctx + + latest_builds = [] + module_builds = sorted(module_builds, key=_key, reverse=True) + for _, ns_builds in groupby( + module_builds, key=lambda x: ":".join([x["name"], x["version"]])): + for _, nsv_builds in groupby( + ns_builds, key=lambda x: x["release"].split(".")[0]): + latest_builds += list(nsv_builds) + break + + # For each latest module build, find the matching ModuleBuild and store its modulemd + # in `mmds`. + mmds = [] + for build in latest_builds: + version, context = build["release"].split(".") + module = models.ModuleBuild.get_build_from_nsvc( + self.db_session, name, stream, version, context) + if not module: + raise ValueError( + "Module %s is tagged in the %s Koji tag, but does not exist " + "in MBS DB." % (":".join([name, stream, version, context]), tag)) + mmds.append(module.mmd()) + + return mmds diff --git a/module_build_service/resolver/LocalResolver.py b/module_build_service/resolver/LocalResolver.py index 6a5a90de..974b8866 100644 --- a/module_build_service/resolver/LocalResolver.py +++ b/module_build_service/resolver/LocalResolver.py @@ -34,12 +34,12 @@ class LocalResolver(DBResolver): backend = "local" - def get_buildrequired_modulemds(self, name, stream, base_module_nsvc): + def get_buildrequired_modulemds(self, name, stream, base_module_mmd): """ Returns modulemd metadata of all module builds with `name` and `stream`. For LocalResolver which is used only for Offline local builds, - the `base_module_nsvc` is ignored. Normally, the `base_module_nsvc is used + the `base_module_mmd` is ignored. Normally, the `base_module_mmd is used to filter out platform:streams which are not compatible with currently used stream version. But during offline local builds, we always have just single platform:stream derived from PLATFORM_ID in /etc/os-release. @@ -50,7 +50,7 @@ class LocalResolver(DBResolver): :param str name: Name of module to return. :param str stream: Stream of module to return. - :param str base_module_nsvc: Ignored in LocalResolver. + :param Modulemd base_module_mmd: Ignored in LocalResolver. :rtype: list :return: List of modulemd metadata. """ diff --git a/module_build_service/resolver/base.py b/module_build_service/resolver/base.py index ebb84e36..803b1219 100644 --- a/module_build_service/resolver/base.py +++ b/module_build_service/resolver/base.py @@ -121,7 +121,7 @@ class GenericResolver(six.with_metaclass(ABCMeta)): raise NotImplementedError() @abstractmethod - def get_buildrequired_modulemds(self, name, stream, base_module_nsvc, strict=False): + def get_buildrequired_modulemds(self, name, stream, base_module_mmd, strict=False): raise NotImplementedError() @abstractmethod diff --git a/module_build_service/utils/mse.py b/module_build_service/utils/mse.py index b312b39f..086e91a9 100644 --- a/module_build_service/utils/mse.py +++ b/module_build_service/utils/mse.py @@ -189,8 +189,7 @@ def _get_mmds_from_requires( if base_module_mmds: for base_module_mmd in base_module_mmds: - base_module_nsvc = base_module_mmd.get_nsvc() - mmds[ns] += resolver.get_buildrequired_modulemds(name, stream, base_module_nsvc) + mmds[ns] += resolver.get_buildrequired_modulemds(name, stream, base_module_mmd) else: mmds[ns] = resolver.get_module_modulemds(name, stream, strict=True) added_mmds[ns] += mmds[ns] diff --git a/setup.py b/setup.py index 7623b71e..031e8109 100644 --- a/setup.py +++ b/setup.py @@ -58,6 +58,7 @@ setup( "mbs = module_build_service.resolver.MBSResolver:MBSResolver", "db = module_build_service.resolver.DBResolver:DBResolver", "local = module_build_service.resolver.LocalResolver:LocalResolver", + "koji = module_build_service.resolver.KojiResolver:KojiResolver" ], }, scripts=["client/mbs-cli"], diff --git a/tests/test_resolver/test_db.py b/tests/test_resolver/test_db.py index 15089967..317f8bf1 100644 --- a/tests/test_resolver/test_db.py +++ b/tests/test_resolver/test_db.py @@ -65,10 +65,9 @@ class TestDBModule: db_session.add(build) db_session.commit() - platform_nsvc = platform_f300103.mmd().get_nsvc() - resolver = mbs_resolver.GenericResolver.create(db_session, tests.conf, backend="db") - result = resolver.get_buildrequired_modulemds("testmodule", "master", platform_nsvc) + result = resolver.get_buildrequired_modulemds( + "testmodule", "master", platform_f300103.mmd()) nsvcs = {m.get_nsvc() for m in result} assert nsvcs == {"testmodule:master:20170109091357:123"} diff --git a/tests/test_resolver/test_koji.py b/tests/test_resolver/test_koji.py new file mode 100644 index 00000000..40d094fb --- /dev/null +++ b/tests/test_resolver/test_koji.py @@ -0,0 +1,214 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza + +import pytest +from mock import patch +from datetime import datetime + +import module_build_service.resolver as mbs_resolver +from module_build_service.utils.general import import_mmd, mmd_to_str, load_mmd +from module_build_service.models import ModuleBuild +import tests + + +@pytest.mark.usefixtures("reuse_component_init_data") +class TestLocalResolverModule: + + def _create_test_modules(self, db_session, koji_tag_with_modules="foo-test"): + mmd = load_mmd(tests.read_staged_data("platform")) + mmd = mmd.copy(mmd.get_module_name(), "f30.1.3") + + import_mmd(db_session, mmd) + platform = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one() + + if koji_tag_with_modules: + platform = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one() + platform_mmd = platform.mmd() + platform_xmd = platform_mmd.get_xmd() + platform_xmd["mbs"]["koji_tag_with_modules"] = koji_tag_with_modules + platform_mmd.set_xmd(platform_xmd) + platform.modulemd = mmd_to_str(platform_mmd) + + for context in ["7c29193d", "7c29193e"]: + mmd = tests.make_module("testmodule:master:20170109091357:" + context) + build = ModuleBuild( + name="testmodule", + stream="master", + version=20170109091357, + state=5, + build_context="dd4de1c346dcf09ce77d38cd4e75094ec1c08ec3", + runtime_context="ec4de1c346dcf09ce77d38cd4e75094ec1c08ef7", + context=context, + koji_tag="module-testmodule-master-20170109091357-" + context, + scmurl="https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79", + batch=3, + owner="Dr. Pepper", + time_submitted=datetime(2018, 11, 15, 16, 8, 18), + time_modified=datetime(2018, 11, 15, 16, 19, 35), + rebuild_strategy="changed-and-after", + modulemd=mmd_to_str(mmd), + ) + build.buildrequires.append(platform) + db_session.add(build) + db_session.commit() + + def test_get_buildrequired_modulemds_fallback_to_db_resolver(self, db_session): + self._create_test_modules(db_session, koji_tag_with_modules=None) + platform = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one() + + resolver = mbs_resolver.GenericResolver.create(db_session, tests.conf, backend="koji") + result = resolver.get_buildrequired_modulemds("testmodule", "master", platform.mmd()) + + nsvcs = {m.get_nsvc() for m in result} + assert nsvcs == { + "testmodule:master:20170109091357:7c29193d", + "testmodule:master:20170109091357:7c29193e"} + + @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession") + def test_get_buildrequired_modulemds_name_not_tagged(self, ClientSession, db_session): + koji_session = ClientSession.return_value + koji_session.getLastEvent.return_value = {"id": 123} + + # No package with such name tagged. + koji_session.listTagged.return_value = [] + + self._create_test_modules(db_session) + platform = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one() + resolver = mbs_resolver.GenericResolver.create(db_session, tests.conf, backend="koji") + result = resolver.get_buildrequired_modulemds("testmodule", "master", platform.mmd()) + + assert result == [] + koji_session.listTagged.assert_called_with( + 'foo-test', inherit=True, package='testmodule', type='module', event=123) + + @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession") + def test_get_buildrequired_modulemds_multiple_streams(self, ClientSession, db_session): + koji_session = ClientSession.return_value + + # We will ask for testmodule:master, but there is also testmodule:2 in a tag. + koji_session.listTagged.return_value = [ + { + 'build_id': 123, 'name': 'testmodule', 'version': '2', + 'release': '820181219174508.9edba152', 'tag_name': 'foo-test' + }, + { + 'build_id': 124, 'name': 'testmodule', 'version': 'master', + 'release': '20170109091357.7c29193d', 'tag_name': 'foo-test' + }] + + self._create_test_modules(db_session) + platform = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one() + resolver = mbs_resolver.GenericResolver.create(db_session, tests.conf, backend="koji") + result = resolver.get_buildrequired_modulemds("testmodule", "master", platform.mmd()) + + nsvcs = {m.get_nsvc() for m in result} + assert nsvcs == {"testmodule:master:20170109091357:7c29193d"} + + @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession") + def test_get_buildrequired_modulemds_tagged_but_not_in_db(self, ClientSession, db_session): + koji_session = ClientSession.return_value + + # We will ask for testmodule:2, but it is not in database, so it should raise + # ValueError later. + koji_session.listTagged.return_value = [ + { + 'build_id': 123, 'name': 'testmodule', 'version': '2', + 'release': '820181219174508.9edba152', 'tag_name': 'foo-test' + }, + { + 'build_id': 124, 'name': 'testmodule', 'version': 'master', + 'release': '20170109091357.7c29193d', 'tag_name': 'foo-test' + }] + + self._create_test_modules(db_session) + platform = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one() + resolver = mbs_resolver.GenericResolver.create(db_session, tests.conf, backend="koji") + expected_error = ("Module testmodule:2:820181219174508:9edba152 is tagged in the " + "foo-test Koji tag, but does not exist in MBS DB.") + with pytest.raises(ValueError, match=expected_error): + resolver.get_buildrequired_modulemds("testmodule", "2", platform.mmd()) + + @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession") + def test_get_buildrequired_modulemds_multiple_versions_contexts( + self, ClientSession, db_session): + koji_session = ClientSession.return_value + + # We will ask for testmodule:2, but it is not in database, so it should raise + # ValueError later. + koji_session.listTagged.return_value = [ + { + 'build_id': 124, 'name': 'testmodule', 'version': 'master', + 'release': '20160110091357.7c29193d', 'tag_name': 'foo-test' + }, + { + 'build_id': 124, 'name': 'testmodule', 'version': 'master', + 'release': '20170109091357.7c29193d', 'tag_name': 'foo-test' + }, + { + 'build_id': 124, 'name': 'testmodule', 'version': 'master', + 'release': '20170109091357.7c29193e', 'tag_name': 'foo-test' + }, + { + 'build_id': 124, 'name': 'testmodule', 'version': 'master', + 'release': '20160109091357.7c29193d', 'tag_name': 'foo-test' + }] + + self._create_test_modules(db_session) + platform = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one() + resolver = mbs_resolver.GenericResolver.create(db_session, tests.conf, backend="koji") + result = resolver.get_buildrequired_modulemds("testmodule", "master", platform.mmd()) + + nsvcs = {m.get_nsvc() for m in result} + assert nsvcs == { + "testmodule:master:20170109091357:7c29193d", + "testmodule:master:20170109091357:7c29193e"} + + @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession") + def test_filter_inherited(self, ClientSession, db_session): + koji_session = ClientSession.return_value + + koji_session.getFullInheritance.return_value = [ + {"name": "foo-test"}, + {"name": "foo-test-parent"}, + ] + + builds = [ + { + 'build_id': 124, 'name': 'testmodule', 'version': 'master', + 'release': '20170110091357.7c29193d', 'tag_name': 'foo-test' + }, + { + 'build_id': 125, 'name': 'testmodule', 'version': 'master', + 'release': '20180109091357.7c29193d', 'tag_name': 'foo-test-parent' + }, + { + 'build_id': 126, 'name': 'testmodule', 'version': '2', + 'release': '20180109091357.7c29193d', 'tag_name': 'foo-test-parent' + }] + + resolver = mbs_resolver.GenericResolver.create(db_session, tests.conf, backend="koji") + new_builds = resolver._filter_inherited(koji_session, builds, "foo-test", {"id": 123}) + + nvrs = {"{name}-{version}-{release}".format(**b) for b in new_builds} + assert nvrs == { + "testmodule-master-20170110091357.7c29193d", + "testmodule-2-20180109091357.7c29193d"}