Fill in the 'filterer_rpms' in backend to not access Koji from frontend.

module_build_service/builder/KojiModuleBuilder.py

@@ -1151,15 +1151,19 @@ chmod 644 %buildroot/etc/rpm/macros.zz-modules
         return weights
 
     @classmethod
-    def get_built_rpms_in_module_build(cls, build):
+    def get_built_rpms_in_module_build(cls, mmd):
         """
-        :param ModuleBuild build: Module build to get the built RPMs from.
+        :param Modulemd mmd: Modulemd to get the built RPMs from.
         :return: list of NVRs
         """
-        koji_session = KojiModuleBuilder.get_session(conf, None)
-        rpms = koji_session.listTaggedRPMS(build.koji_tag, latest=True)[0]
-        nvrs = set(kobo.rpmlib.make_nvr(rpm, force_epoch=True) for rpm in rpms)
-        return list(nvrs)
+        with models.make_session(conf) as db_session:
+            build = models.ModuleBuild.get_build_from_nsvc(
+                db_session, mmd.get_name(), mmd.get_stream(), mmd.get_version(),
+                mmd.get_context())
+            koji_session = KojiModuleBuilder.get_session(conf, None)
+            rpms = koji_session.listTaggedRPMS(build.koji_tag, latest=True)[0]
+            nvrs = set(kobo.rpmlib.make_nvr(rpm, force_epoch=True) for rpm in rpms)
+            return list(nvrs)
 
     def finalize(self):
         # Only import to koji CG if the module is "done".

module_build_service/builder/base.py

@@ -337,9 +337,9 @@ class GenericBuilder(six.with_metaclass(ABCMeta)):
         raise NotImplementedError()
 
     @classmethod
-    def get_built_rpms_in_module_build(cls, build):
+    def get_built_rpms_in_module_build(cls, mmd):
         """
-        :param ModuleBuild build: Module build to get the built RPMs from.
+        :param Modulemd mmd: Modulemd to get the built RPMs from.
         :return: list of NVRs
         """
         raise NotImplementedError()

module_build_service/resolver/DBResolver.py

@@ -22,8 +22,6 @@
 # Written by Matt Prahl <mprahl@redhat.com>
 #            Jan Kaluza <jkaluza@redhat.com>
 
-import kobo.rpmlib
-
 from module_build_service import log
 from module_build_service.resolver.base import GenericResolver
 from module_build_service import models
@@ -173,14 +171,13 @@ class DBResolver(GenericResolver):
         """
         Resolves the requires list of N:S or N:S:V:C to a dictionary with keys as
         the module name and the values as a dictionary with keys of ref,
-        stream, version, filtered_rpms.
+        stream, version.
         If there are some modules loaded by utils.load_local_builds(...), these
         local modules will be considered when resolving the requires. A RuntimeError
         is raised on DB lookup errors.
         :param requires: a dictionary with the module name as the key and the stream as the value
         :return: a dictionary
         """
-        from module_build_service.builder import GenericBuilder
         new_requires = {}
         with models.make_session(self.config) as session:
             for nsvc in requires:
@@ -203,11 +200,7 @@ class DBResolver(GenericResolver):
                         'ref': None,
                         'stream': local_build.stream,
                         'version': local_build.version,
-                        'context': local_build.context,
-                        # No need to set filtered_rpms for local builds, because MBS
-                        # filters the RPMs automatically when the module build is
-                        # done.
-                        'filtered_rpms': []
+                        'context': local_build.context
                     }
                     continue
 
@@ -222,7 +215,6 @@ class DBResolver(GenericResolver):
                     raise UnprocessableEntity('The module {} was not found'.format(nsvc))
 
                 commit_hash = None
-                filtered_rpms = []
                 mmd = build.mmd()
                 mbs_xmd = mmd.get_xmd().get('mbs')
                 if mbs_xmd and 'commit' in mbs_xmd.keys():
@@ -237,22 +229,11 @@ class DBResolver(GenericResolver):
                         'The module "{}" is not built using Module Stream Expansion. '
                         'Please rebuild this module first'.format(nsvc))
 
-                # Find out the particular NVR of filtered packages
-                rpm_filter = mmd.get_rpm_filter()
-                if rpm_filter and rpm_filter.get():
-                    rpm_filter = rpm_filter.get()
-                    built_nvrs = GenericBuilder.get_built_rpms_in_module_build(build)
-                    for nvr in built_nvrs:
-                        parsed_nvr = kobo.rpmlib.parse_nvr(nvr)
-                        if parsed_nvr["name"] in rpm_filter:
-                            filtered_rpms.append(nvr)
-
                 new_requires[module_name] = {
                     'ref': commit_hash,
                     'stream': module_stream,
                     'version': build.version,
-                    'context': build.context,
-                    'filtered_rpms': filtered_rpms,
+                    'context': build.context
                 }
 
         return new_requires

module_build_service/resolver/MBSResolver.py

@@ -266,8 +266,7 @@ class MBSResolver(GenericResolver):
             "module_name": {
                 "ref": module_commit_hash,
                 "stream": original_module_stream,
-                "version": module_version,
-                "filtered_rpms": ["nvr", ...]
+                "version": module_version
             },
             ...
         }

module_build_service/scheduler/handlers/modules.py

@@ -32,7 +32,8 @@ from module_build_service.utils import (
     attempt_to_reuse_all_components,
     record_component_builds,
     get_rpm_release,
-    generate_koji_tag)
+    generate_koji_tag,
+    record_filtered_rpms)
 from module_build_service.errors import UnprocessableEntity, Forbidden, ValidationError
 
 from requests.exceptions import ConnectionError
@@ -151,6 +152,7 @@ def init(config, session, msg):
     try:
         mmd = build.mmd()
         record_component_builds(mmd, build, session=session)
+        mmd = record_filtered_rpms(mmd)
         build.modulemd = mmd.dumps()
         build.transition(conf, models.BUILD_STATES["wait"])
     # Catch custom exceptions that we can expose to the user

module_build_service/utils/submit.py

@@ -29,6 +29,7 @@ import tempfile
 import os
 from multiprocessing.dummy import Pool as ThreadPool
 from datetime import datetime
+import kobo.rpmlib
 
 from module_build_service import conf, db, log, models, Modulemd
 from module_build_service.errors import (
@@ -38,6 +39,53 @@ import module_build_service.scm
 from .mse import generate_expanded_mmds
 
 
+def record_filtered_rpms(mmd):
+    """
+    Reads the mmd["xmd"]["buildrequires"] and extends it with "filtered_rpms"
+    list containing the NVRs of filtered RPMs in a buildrequired module.
+
+    :param Modulemd mmd: Modulemd of input module.
+    :rtype: Modulemd
+    :return: Modulemd extended with the "filtered_rpms" in XMD section.
+    """
+    # Imported here to allow import of utils in GenericBuilder.
+    from module_build_service.builder import GenericBuilder
+
+    new_buildrequires = {}
+    resolver = module_build_service.resolver.GenericResolver.create(conf)
+    for req_name, req_data in mmd.get_xmd()["mbs"]["buildrequires"].items():
+        # In case this is module resubmit or local build, the filtered_rpms
+        # will already be there, so there is no point in generating them again.
+        if "filtered_rpms" in req_data:
+            continue
+
+        # We can just get the first modulemd data from result right here thanks to
+        # strict=True, so in case the module cannot be found, get_module_modulemds
+        # raises an exception.
+        req_mmd = resolver.get_module_modulemds(
+            req_name, req_data["stream"], req_data["version"], req_data["context"], True)[0]
+
+        # Find out the particular NVR of filtered packages
+        filtered_rpms = []
+        rpm_filter = req_mmd.get_rpm_filter()
+        if rpm_filter and rpm_filter.get():
+            rpm_filter = rpm_filter.get()
+            built_nvrs = GenericBuilder.backends[conf.system].get_built_rpms_in_module_build(
+                req_mmd)
+            for nvr in built_nvrs:
+                parsed_nvr = kobo.rpmlib.parse_nvr(nvr)
+                if parsed_nvr["name"] in rpm_filter:
+                    filtered_rpms.append(nvr)
+        req_data["filtered_rpms"] = filtered_rpms
+        new_buildrequires[req_name] = req_data
+
+    # Replace the old buildrequires with new ones.
+    xmd = glib.from_variant_dict(mmd.get_xmd())
+    xmd["mbs"]["buildrequires"] = new_buildrequires
+    mmd.set_xmd(glib.dict_values(xmd))
+    return mmd
+
+
 def _scm_get_latest(pkg):
     try:
         # If the modulemd specifies that the 'f25' branch is what

tests/staged_data/includedmodules.yaml

@@ -18,6 +18,9 @@ data:
     api:
         rpms:
             - bash
+    xmd:
+        mbs:
+            buildrequires: {}
     components:
         rpms:
             file:

tests/staged_data/testmodule_init.yaml

@@ -0,0 +1,45 @@
+document: modulemd
+version: 1
+data:
+    summary: A test module in all its beautiful beauty
+    description: >-
+        This module demonstrates how to write simple modulemd files And
+        can be used for testing the build and release pipeline.
+    license:
+        module: [ MIT ]
+    dependencies:
+        buildrequires:
+            platform: f28
+        requires:
+            platform: f28
+    references:
+        community: https://docs.pagure.org/modularity/
+        documentation: https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules
+    profiles:
+        default:
+            rpms:
+            - tangerine
+    xmd:
+        mbs:
+            buildrequires:
+               platform:
+                   ref: virtual
+                   stream: f28
+                   version: '3'
+                   context: '00000000'
+    api:
+        rpms:
+        - perl-Tangerine
+        - tangerine
+    components:
+        rpms:
+            perl-List-Compare:
+                rationale: A dependency of tangerine.
+                ref: master
+            perl-Tangerine:
+                rationale: Provides API for this module and is a dependency of tangerine.
+                ref: master
+            tangerine:
+                rationale: Provides API for this module.
+                buildorder: 10
+                ref: master

tests/test_builder/test_koji.py

@@ -32,7 +32,7 @@ import module_build_service.messaging
 import module_build_service.scheduler.handlers.repos
 import module_build_service.models
 import module_build_service.builder
-from module_build_service import glib
+from module_build_service import glib, db
 
 import pytest
 from mock import patch, MagicMock
@@ -545,7 +545,17 @@ class TestKojiBuilder:
              'size': 878684}], [])
         get_session.return_value = session
 
-        ret = KojiModuleBuilder.get_built_rpms_in_module_build(self.module)
+        # Module builds generated by init_data uses generic modulemd file and
+        # the module's name/stream/version/context does not have to match it.
+        # But for this test, we need it to match.
+        mmd = self.module.mmd()
+        self.module.name = mmd.get_name()
+        self.module.stream = mmd.get_stream()
+        self.module.version = mmd.get_version()
+        self.module.context = mmd.get_context()
+        db.session.commit()
+
+        ret = KojiModuleBuilder.get_built_rpms_in_module_build(mmd)
         assert set(ret) == set(
             ['bar-2:1.30-4.el8+1308+551bfa71', 'tar-2:1.30-4.el8+1308+551bfa71'])
 

tests/test_resolver/test_db.py

@@ -120,25 +120,8 @@ class TestDBModule:
             ]
             assert set(result) == set(expected)
 
-    @patch("module_build_service.builder.base.GenericBuilder.get_built_rpms_in_module_build")
-    def test_resolve_requires(self, built_rpms):
+    def test_resolve_requires(self):
         build = models.ModuleBuild.query.get(2)
-        mmd = build.mmd()
-        filter_list = Modulemd.SimpleSet()
-        filter_list.add("foo")
-        filter_list.add("bar")
-        mmd.set_rpm_filter(filter_list)
-        build.modulemd = mmd.dumps()
-        db.session.commit()
-
-        built_rpms.return_value = [
-            "foo-0:2.4.48-3.el8+1308+551bfa71",
-            "foo-debuginfo-0:2.4.48-3.el8+1308+551bfa71",
-            "bar-0:2.5.48-3.el8+1308+551bfa71",
-            "bar-debuginfo-0:2.5.48-3.el8+1308+551bfa71",
-            "x-0:2.5.48-3.el8+1308+551bfa71",
-            "x-debuginfo-0:2.5.48-3.el8+1308+551bfa71"]
-
         resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db')
         result = resolver.resolve_requires([":".join([
             build.name, build.stream, build.version, build.context])])
@@ -146,10 +129,7 @@ class TestDBModule:
         assert result == {
             'testmodule': {
                 'stream': 'master', 'version': '20170109091357', 'context': u'78e4a6fd',
-                'ref': 'ff1ea79fc952143efeed1851aa0aa006559239ba',
-                'filtered_rpms': [
-                    'foo-0:2.4.48-3.el8+1308+551bfa71',
-                    'bar-0:2.5.48-3.el8+1308+551bfa71']}}
+                'ref': 'ff1ea79fc952143efeed1851aa0aa006559239ba'}}
 
     def test_resolve_profiles(self):
         """

tests/test_scheduler/test_module_init.py

@@ -28,7 +28,7 @@ from tests import conf, clean_database
 from tests.test_views.test_views import FakeSCM
 import module_build_service.messaging
 import module_build_service.scheduler.handlers.modules
-from module_build_service import build_logs
+from module_build_service import build_logs, Modulemd, db
 from module_build_service.models import make_session, ModuleBuild, ComponentBuild
 
 
@@ -39,7 +39,7 @@ class TestModuleInit:
         self.staged_data_dir = os.path.join(
             os.path.dirname(__file__), '../', 'staged_data')
         testmodule_yml_path = os.path.join(
-            self.staged_data_dir, 'testmodule.yaml')
+            self.staged_data_dir, 'testmodule_init.yaml')
         with open(testmodule_yml_path, 'r') as f:
             yaml = f.read()
         scmurl = 'git://pkgs.domain.local/modules/testmodule?#620ec77'
@@ -55,19 +55,43 @@ class TestModuleInit:
         except Exception:
             pass
 
+    @patch("module_build_service.builder.KojiModuleBuilder.KojiModuleBuilder."
+           "get_built_rpms_in_module_build")
     @patch('module_build_service.scm.SCM')
-    def test_init_basic(self, mocked_scm):
-        FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
+    def test_init_basic(self, mocked_scm, built_rpms):
+        FakeSCM(mocked_scm, 'testmodule', 'testmodule_init.yaml',
                 '620ec77321b2ea7b0d67d82992dda3e1d67055b4')
+
+        built_rpms.return_value = [
+            "foo-0:2.4.48-3.el8+1308+551bfa71",
+            "foo-debuginfo-0:2.4.48-3.el8+1308+551bfa71",
+            "bar-0:2.5.48-3.el8+1308+551bfa71",
+            "bar-debuginfo-0:2.5.48-3.el8+1308+551bfa71",
+            "x-0:2.5.48-3.el8+1308+551bfa71",
+            "x-debuginfo-0:2.5.48-3.el8+1308+551bfa71"]
+
+        platform_build = ModuleBuild.query.get(1)
+        mmd = platform_build.mmd()
+        filter_list = Modulemd.SimpleSet()
+        filter_list.add("foo")
+        filter_list.add("bar")
+        mmd.set_rpm_filter(filter_list)
+        platform_build.modulemd = mmd.dumps()
+        db.session.commit()
+
         msg = module_build_service.messaging.MBSModule(
             msg_id=None, module_build_id=2, module_build_state='init')
+
         with make_session(conf) as session:
             self.fn(config=conf, session=session, msg=msg)
         build = ModuleBuild.query.filter_by(id=2).one()
         # Make sure the module entered the wait state
         assert build.state == 1, build.state
         # Make sure format_mmd was run properly
-        assert type(build.mmd().get_xmd()['mbs']) is GLib.Variant
+        xmd_mbs = build.mmd().get_xmd()['mbs']
+        assert type(xmd_mbs) is GLib.Variant
+        assert xmd_mbs["buildrequires"]["platform"]["filtered_rpms"] == [
+            'foo-0:2.4.48-3.el8+1308+551bfa71', 'bar-0:2.5.48-3.el8+1308+551bfa71']
 
     @patch('module_build_service.scm.SCM')
     def test_init_scm_not_available(self, mocked_scm):
@@ -90,7 +114,7 @@ class TestModuleInit:
            new_callable=PropertyMock, return_value=True)
     @patch('module_build_service.scm.SCM')
     def test_init_includedmodule(self, mocked_scm, mocked_mod_allow_repo):
-        FakeSCM(mocked_scm, "includedmodules", ['testmodule.yaml'])
+        FakeSCM(mocked_scm, "includedmodules", ['testmodule_init.yaml'])
         includedmodules_yml_path = os.path.join(
             self.staged_data_dir, 'includedmodules.yaml')
         with open(includedmodules_yml_path, 'r') as f: