Add initial code for KojiResolver class.

This commit:

- Adds KojiResolver class and KojiResolver tests.
- Changes the GenericResolver and its subclasses to pass base_module_mmds
  instead of base_module_nsvc to get_buildrequired_modulemds. This is needed,
  because KojiResolver needs to access XMD section of base module.
- Implements KojiResolver.get_buildrequired_modulemds to ask Koji for list of
  modules tagged in the Koji tag and return their modulemds.

docs/VIRTUAL_MODULES.rst

@@ -66,6 +66,9 @@ Custom fields in xmd:
 - ``koji_tag`` - this defines the Koji tag with the RPMs that are part of this module. For base
   modules this will likely be a tag representing a buildroot. If this is a metadata-only module,
   then this can be left unset.
+- ``koji_tag_with_modules`` - this defines the Koji tag with the module builds. These modules are
+  later used to fulfill the build requirements of modules built on against this module. This
+  option is used only when ``KojiResolver`` is enabled on the MBS server.
 - ``virtual_streams`` - the list of streams which groups multiple modules together. For more
   information on this field, see the ``Virtual Streams`` section below.
 - ``disttag_marking`` - if this module is a base module, then MBS will use the stream of the base

module_build_service/resolver/DBResolver.py

@@ -155,19 +155,19 @@ class DBResolver(GenericResolver):
 
         return [build.mmd() for build in builds]
 
-    def get_buildrequired_modulemds(self, name, stream, base_module_nsvc):
+    def get_buildrequired_modulemds(self, name, stream, base_module_mmd):
         """
         Returns modulemd metadata of all module builds with `name` and `stream` buildrequiring
-        base module defined by `base_module_nsvc` NSVC.
+        base module defined by `base_module_mmd` NSVC.
 
         :param str name: Name of module to return.
         :param str stream: Stream of module to return.
-        :param str base_module_nsvc: NSVC of base module which must be buildrequired by returned
+        :param Modulemd base_module_mmd: NSVC of base module which must be buildrequired by returned
             modules.
         :rtype: list
         :return: List of modulemd metadata.
         """
-        log.debug("Looking for %s:%s buildrequiring %s", name, stream, base_module_nsvc)
+        log.debug("Looking for %s:%s buildrequiring %s", name, stream, base_module_mmd.get_nsvc())
         query = self.db_session.query(models.ModuleBuild)
         query = query.filter_by(name=name, stream=stream, state=models.BUILD_STATES["ready"])
 
@@ -182,8 +182,8 @@ class DBResolver(GenericResolver):
         query = query.join(mb_to_br, mb_to_br.c.module_id == models.ModuleBuild.id).join(
             module_br_alias, mb_to_br.c.module_buildrequire_id == module_br_alias.id)
 
-        # Get only modules buildrequiring particular base_module_nsvc
-        n, s, v, c = base_module_nsvc.split(":")
+        # Get only modules buildrequiring particular base_module_mmd
+        n, s, v, c = base_module_mmd.get_nsvc().split(":")
         query = query.filter(
             module_br_alias.name == n,
             module_br_alias.stream == s,

module_build_service/resolver/KojiResolver.py

@@ -0,0 +1,162 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019  Red Hat, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+# Written by Jan Kaluza <jkaluza@redhat.com>
+
+from itertools import groupby
+
+from module_build_service.resolver.DBResolver import DBResolver
+from module_build_service import conf, models, log
+
+
+class KojiResolver(DBResolver):
+    """
+    Resolver using Koji server running in infrastructure.
+    """
+
+    backend = "koji"
+
+    def _filter_inherited(self, koji_session, module_builds, top_tag, event):
+        """
+        Look at the tag inheritance and keep builds only from the topmost tag.
+
+        For example, we have "foo:bar:1" and "foo:bar:2" builds. We also have "foo-tag" which
+        inherits "foo-parent-tag". The "foo:bar:1" is tagged in the "foo-tag". The "foo:bar:2"
+        is tagged in the "foo-parent-tag".
+
+        In this case, this function filters out the foo:bar:2, because "foo:bar:1" is tagged
+        lower in the inheritance tree in the "foo-tag".
+
+        For normal RPMs, using latest=True for listTagged() call, Koji would automatically do
+        this, but it does not understand streams, so we have to reimplement it here.
+
+        :param KojiSession koji_session: Koji session.
+        :param list module_builds: List of builds as returned by KojiSession.listTagged method.
+        :param str top_tag: The top Koji tag.
+        :param dict event: Koji event defining the time at which the `module_builds` have been
+            fetched.
+        :return list: Filtered list of builds.
+        """
+        inheritance = [
+            tag["name"] for tag in koji_session.getFullInheritance(top_tag, event=event["id"])
+        ]
+
+        def keyfunc(mb):
+            return (mb["name"], mb["version"])
+
+        result = []
+
+        # Group modules by Name-Stream
+        for _, builds in groupby(sorted(module_builds, key=keyfunc), keyfunc):
+            builds = list(builds)
+            # For each N-S combination find out which tags it's in
+            available_in = set(build["tag_name"] for build in builds)
+
+            # And find out which is the topmost tag
+            for tag in [top_tag] + inheritance:
+                if tag in available_in:
+                    break
+
+            # And keep only builds from that topmost tag
+            result.extend(build for build in builds if build["tag_name"] == tag)
+
+        return result
+
+    def get_buildrequired_modulemds(self, name, stream, base_module_mmd):
+        """
+        Returns modulemd metadata of all module builds with `name` and `stream` which are tagged
+        in the Koji tag defined in `base_module_mmd`.
+
+        :param str name: Name of module to return.
+        :param str stream: Stream of module to return.
+        :param Modulemd base_module_mmd: Base module metadata.
+        :return list: List of modulemd metadata.
+        """
+        # Get the `koji_tag_with_modules`. If the `koji_tag_with_modules` is not configured for
+        # the base module, fallback to DBResolver.
+        tag = base_module_mmd.get_xmd().get("mbs", {}).get("koji_tag_with_modules")
+        if not tag:
+            log.info(
+                "The %s does not define 'koji_tag_with_modules'. Falling back to DBResolver." % (
+                    base_module_mmd.get_nsvc()))
+            return DBResolver.get_buildrequired_modulemds(self, name, stream, base_module_mmd)
+
+        # Create KojiSession. We need to import here because of circular dependencies.
+        from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder
+        koji_session = KojiModuleBuilder.get_session(conf, login=False)
+        event = koji_session.getLastEvent()
+
+        # List all the modular builds in the modular Koji tag.
+        # We cannot use latest=True here, because we need to get all the
+        # available streams of all modules. The stream is represented as
+        # "version" in Koji build and with latest=True, Koji would return
+        # only builds with the highest version.
+        # We also cannot ask for particular `stream`, because Koji does not support that.
+        module_builds = koji_session.listTagged(
+            tag, inherit=True, type="module", package=name, event=event["id"])
+
+        # Filter out different streams
+        normalized_stream = stream.replace("-", "_")
+        module_builds = [b for b in module_builds if b["version"] == normalized_stream]
+
+        # Filter out builds inherited from non-top tag
+        module_builds = self._filter_inherited(koji_session, module_builds, tag, event)
+
+        # Find the latest builds of all modules. This does the following:
+        # - Sorts the module_builds descending by Koji NVR (which maps to NSV
+        #   for modules). Split release into modular version and context, and
+        #   treat version as numeric.
+        # - Groups the sorted module_builds by NV (NS in modular world).
+        #   In each resulting `ns_group`, the first item is actually build
+        #   with the latest version (because the list is still sorted by NVR).
+        # - Groups the `ns_group` again by "release" ("version" in modular
+        #   world) to just get all the "contexts" of the given NSV. This is
+        #   stored in `nsv_builds`.
+        # - The `nsv_builds` contains the builds representing all the contexts
+        #   of the latest version for give name-stream, so add them to
+        #   `latest_builds`.
+        def _key(build):
+            ver, ctx = build["release"].split(".", 1)
+            return build["name"], build["version"], int(ver), ctx
+
+        latest_builds = []
+        module_builds = sorted(module_builds, key=_key, reverse=True)
+        for _, ns_builds in groupby(
+                module_builds, key=lambda x: ":".join([x["name"], x["version"]])):
+            for _, nsv_builds in groupby(
+                    ns_builds, key=lambda x: x["release"].split(".")[0]):
+                latest_builds += list(nsv_builds)
+                break
+
+        # For each latest module build, find the matching ModuleBuild and store its modulemd
+        # in `mmds`.
+        mmds = []
+        for build in latest_builds:
+            version, context = build["release"].split(".")
+            module = models.ModuleBuild.get_build_from_nsvc(
+                self.db_session, name, stream, version, context)
+            if not module:
+                raise ValueError(
+                    "Module %s is tagged in the %s Koji tag, but does not exist "
+                    "in MBS DB." % (":".join([name, stream, version, context]), tag))
+            mmds.append(module.mmd())
+
+        return mmds

module_build_service/resolver/LocalResolver.py

@@ -34,12 +34,12 @@ class LocalResolver(DBResolver):
 
     backend = "local"
 
-    def get_buildrequired_modulemds(self, name, stream, base_module_nsvc):
+    def get_buildrequired_modulemds(self, name, stream, base_module_mmd):
         """
         Returns modulemd metadata of all module builds with `name` and `stream`.
 
         For LocalResolver which is used only for Offline local builds,
-        the `base_module_nsvc` is ignored. Normally, the `base_module_nsvc is used
+        the `base_module_mmd` is ignored. Normally, the `base_module_mmd is used
         to filter out platform:streams which are not compatible with currently used
         stream version. But during offline local builds, we always have just single
         platform:stream derived from PLATFORM_ID in /etc/os-release.
@@ -50,7 +50,7 @@ class LocalResolver(DBResolver):
 
         :param str name: Name of module to return.
         :param str stream: Stream of module to return.
-        :param str base_module_nsvc: Ignored in LocalResolver.
+        :param Modulemd base_module_mmd: Ignored in LocalResolver.
         :rtype: list
         :return: List of modulemd metadata.
         """

module_build_service/resolver/base.py

@@ -121,7 +121,7 @@ class GenericResolver(six.with_metaclass(ABCMeta)):
         raise NotImplementedError()
 
     @abstractmethod
-    def get_buildrequired_modulemds(self, name, stream, base_module_nsvc, strict=False):
+    def get_buildrequired_modulemds(self, name, stream, base_module_mmd, strict=False):
         raise NotImplementedError()
 
     @abstractmethod

module_build_service/utils/mse.py

@@ -189,8 +189,7 @@ def _get_mmds_from_requires(
 
             if base_module_mmds:
                 for base_module_mmd in base_module_mmds:
-                    base_module_nsvc = base_module_mmd.get_nsvc()
-                    mmds[ns] += resolver.get_buildrequired_modulemds(name, stream, base_module_nsvc)
+                    mmds[ns] += resolver.get_buildrequired_modulemds(name, stream, base_module_mmd)
             else:
                 mmds[ns] = resolver.get_module_modulemds(name, stream, strict=True)
             added_mmds[ns] += mmds[ns]

setup.py

@@ -58,6 +58,7 @@ setup(
             "mbs = module_build_service.resolver.MBSResolver:MBSResolver",
             "db = module_build_service.resolver.DBResolver:DBResolver",
             "local = module_build_service.resolver.LocalResolver:LocalResolver",
+            "koji = module_build_service.resolver.KojiResolver:KojiResolver"
         ],
     },
     scripts=["client/mbs-cli"],

tests/test_resolver/test_db.py

@@ -65,10 +65,9 @@ class TestDBModule:
         db_session.add(build)
         db_session.commit()
 
-        platform_nsvc = platform_f300103.mmd().get_nsvc()
-
         resolver = mbs_resolver.GenericResolver.create(db_session, tests.conf, backend="db")
-        result = resolver.get_buildrequired_modulemds("testmodule", "master", platform_nsvc)
+        result = resolver.get_buildrequired_modulemds(
+            "testmodule", "master", platform_f300103.mmd())
         nsvcs = {m.get_nsvc() for m in result}
         assert nsvcs == {"testmodule:master:20170109091357:123"}
 

tests/test_resolver/test_koji.py

@@ -0,0 +1,214 @@
+# Copyright (c) 2019  Red Hat, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+# Written by Jan Kaluza <jkaluza@redhat.com>
+
+import pytest
+from mock import patch
+from datetime import datetime
+
+import module_build_service.resolver as mbs_resolver
+from module_build_service.utils.general import import_mmd, mmd_to_str, load_mmd
+from module_build_service.models import ModuleBuild
+import tests
+
+
+@pytest.mark.usefixtures("reuse_component_init_data")
+class TestLocalResolverModule:
+
+    def _create_test_modules(self, db_session, koji_tag_with_modules="foo-test"):
+        mmd = load_mmd(tests.read_staged_data("platform"))
+        mmd = mmd.copy(mmd.get_module_name(), "f30.1.3")
+
+        import_mmd(db_session, mmd)
+        platform = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one()
+
+        if koji_tag_with_modules:
+            platform = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one()
+            platform_mmd = platform.mmd()
+            platform_xmd = platform_mmd.get_xmd()
+            platform_xmd["mbs"]["koji_tag_with_modules"] = koji_tag_with_modules
+            platform_mmd.set_xmd(platform_xmd)
+            platform.modulemd = mmd_to_str(platform_mmd)
+
+        for context in ["7c29193d", "7c29193e"]:
+            mmd = tests.make_module("testmodule:master:20170109091357:" + context)
+            build = ModuleBuild(
+                name="testmodule",
+                stream="master",
+                version=20170109091357,
+                state=5,
+                build_context="dd4de1c346dcf09ce77d38cd4e75094ec1c08ec3",
+                runtime_context="ec4de1c346dcf09ce77d38cd4e75094ec1c08ef7",
+                context=context,
+                koji_tag="module-testmodule-master-20170109091357-" + context,
+                scmurl="https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79",
+                batch=3,
+                owner="Dr. Pepper",
+                time_submitted=datetime(2018, 11, 15, 16, 8, 18),
+                time_modified=datetime(2018, 11, 15, 16, 19, 35),
+                rebuild_strategy="changed-and-after",
+                modulemd=mmd_to_str(mmd),
+            )
+            build.buildrequires.append(platform)
+            db_session.add(build)
+        db_session.commit()
+
+    def test_get_buildrequired_modulemds_fallback_to_db_resolver(self, db_session):
+        self._create_test_modules(db_session, koji_tag_with_modules=None)
+        platform = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one()
+
+        resolver = mbs_resolver.GenericResolver.create(db_session, tests.conf, backend="koji")
+        result = resolver.get_buildrequired_modulemds("testmodule", "master", platform.mmd())
+
+        nsvcs = {m.get_nsvc() for m in result}
+        assert nsvcs == {
+            "testmodule:master:20170109091357:7c29193d",
+            "testmodule:master:20170109091357:7c29193e"}
+
+    @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")
+    def test_get_buildrequired_modulemds_name_not_tagged(self, ClientSession, db_session):
+        koji_session = ClientSession.return_value
+        koji_session.getLastEvent.return_value = {"id": 123}
+
+        # No package with such name tagged.
+        koji_session.listTagged.return_value = []
+
+        self._create_test_modules(db_session)
+        platform = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one()
+        resolver = mbs_resolver.GenericResolver.create(db_session, tests.conf, backend="koji")
+        result = resolver.get_buildrequired_modulemds("testmodule", "master", platform.mmd())
+
+        assert result == []
+        koji_session.listTagged.assert_called_with(
+            'foo-test', inherit=True, package='testmodule', type='module', event=123)
+
+    @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")
+    def test_get_buildrequired_modulemds_multiple_streams(self, ClientSession, db_session):
+        koji_session = ClientSession.return_value
+
+        # We will ask for testmodule:master, but there is also testmodule:2 in a tag.
+        koji_session.listTagged.return_value = [
+            {
+                'build_id': 123, 'name': 'testmodule', 'version': '2',
+                'release': '820181219174508.9edba152', 'tag_name': 'foo-test'
+            },
+            {
+                'build_id': 124, 'name': 'testmodule', 'version': 'master',
+                'release': '20170109091357.7c29193d', 'tag_name': 'foo-test'
+            }]
+
+        self._create_test_modules(db_session)
+        platform = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one()
+        resolver = mbs_resolver.GenericResolver.create(db_session, tests.conf, backend="koji")
+        result = resolver.get_buildrequired_modulemds("testmodule", "master", platform.mmd())
+
+        nsvcs = {m.get_nsvc() for m in result}
+        assert nsvcs == {"testmodule:master:20170109091357:7c29193d"}
+
+    @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")
+    def test_get_buildrequired_modulemds_tagged_but_not_in_db(self, ClientSession, db_session):
+        koji_session = ClientSession.return_value
+
+        # We will ask for testmodule:2, but it is not in database, so it should raise
+        # ValueError later.
+        koji_session.listTagged.return_value = [
+            {
+                'build_id': 123, 'name': 'testmodule', 'version': '2',
+                'release': '820181219174508.9edba152', 'tag_name': 'foo-test'
+            },
+            {
+                'build_id': 124, 'name': 'testmodule', 'version': 'master',
+                'release': '20170109091357.7c29193d', 'tag_name': 'foo-test'
+            }]
+
+        self._create_test_modules(db_session)
+        platform = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one()
+        resolver = mbs_resolver.GenericResolver.create(db_session, tests.conf, backend="koji")
+        expected_error = ("Module testmodule:2:820181219174508:9edba152 is tagged in the "
+                          "foo-test Koji tag, but does not exist in MBS DB.")
+        with pytest.raises(ValueError, match=expected_error):
+            resolver.get_buildrequired_modulemds("testmodule", "2", platform.mmd())
+
+    @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")
+    def test_get_buildrequired_modulemds_multiple_versions_contexts(
+            self, ClientSession, db_session):
+        koji_session = ClientSession.return_value
+
+        # We will ask for testmodule:2, but it is not in database, so it should raise
+        # ValueError later.
+        koji_session.listTagged.return_value = [
+            {
+                'build_id': 124, 'name': 'testmodule', 'version': 'master',
+                'release': '20160110091357.7c29193d', 'tag_name': 'foo-test'
+            },
+            {
+                'build_id': 124, 'name': 'testmodule', 'version': 'master',
+                'release': '20170109091357.7c29193d', 'tag_name': 'foo-test'
+            },
+            {
+                'build_id': 124, 'name': 'testmodule', 'version': 'master',
+                'release': '20170109091357.7c29193e', 'tag_name': 'foo-test'
+            },
+            {
+                'build_id': 124, 'name': 'testmodule', 'version': 'master',
+                'release': '20160109091357.7c29193d', 'tag_name': 'foo-test'
+            }]
+
+        self._create_test_modules(db_session)
+        platform = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one()
+        resolver = mbs_resolver.GenericResolver.create(db_session, tests.conf, backend="koji")
+        result = resolver.get_buildrequired_modulemds("testmodule", "master", platform.mmd())
+
+        nsvcs = {m.get_nsvc() for m in result}
+        assert nsvcs == {
+            "testmodule:master:20170109091357:7c29193d",
+            "testmodule:master:20170109091357:7c29193e"}
+
+    @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")
+    def test_filter_inherited(self, ClientSession, db_session):
+        koji_session = ClientSession.return_value
+
+        koji_session.getFullInheritance.return_value = [
+            {"name": "foo-test"},
+            {"name": "foo-test-parent"},
+        ]
+
+        builds = [
+            {
+                'build_id': 124, 'name': 'testmodule', 'version': 'master',
+                'release': '20170110091357.7c29193d', 'tag_name': 'foo-test'
+            },
+            {
+                'build_id': 125, 'name': 'testmodule', 'version': 'master',
+                'release': '20180109091357.7c29193d', 'tag_name': 'foo-test-parent'
+            },
+            {
+                'build_id': 126, 'name': 'testmodule', 'version': '2',
+                'release': '20180109091357.7c29193d', 'tag_name': 'foo-test-parent'
+            }]
+
+        resolver = mbs_resolver.GenericResolver.create(db_session, tests.conf, backend="koji")
+        new_builds = resolver._filter_inherited(koji_session, builds, "foo-test", {"id": 123})
+
+        nvrs = {"{name}-{version}-{release}".format(**b) for b in new_builds}
+        assert nvrs == {
+            "testmodule-master-20170110091357.7c29193d",
+            "testmodule-2-20180109091357.7c29193d"}