diff --git a/module_build_service/builder/KojiModuleBuilder.py b/module_build_service/builder/KojiModuleBuilder.py index c3e388e1..a406def6 100644 --- a/module_build_service/builder/KojiModuleBuilder.py +++ b/module_build_service/builder/KojiModuleBuilder.py @@ -1151,15 +1151,19 @@ chmod 644 %buildroot/etc/rpm/macros.zz-modules return weights @classmethod - def get_built_rpms_in_module_build(cls, build): + def get_built_rpms_in_module_build(cls, mmd): """ - :param ModuleBuild build: Module build to get the built RPMs from. + :param Modulemd mmd: Modulemd to get the built RPMs from. :return: list of NVRs """ - koji_session = KojiModuleBuilder.get_session(conf, None) - rpms = koji_session.listTaggedRPMS(build.koji_tag, latest=True)[0] - nvrs = set(kobo.rpmlib.make_nvr(rpm, force_epoch=True) for rpm in rpms) - return list(nvrs) + with models.make_session(conf) as db_session: + build = models.ModuleBuild.get_build_from_nsvc( + db_session, mmd.get_name(), mmd.get_stream(), mmd.get_version(), + mmd.get_context()) + koji_session = KojiModuleBuilder.get_session(conf, None) + rpms = koji_session.listTaggedRPMS(build.koji_tag, latest=True)[0] + nvrs = set(kobo.rpmlib.make_nvr(rpm, force_epoch=True) for rpm in rpms) + return list(nvrs) def finalize(self): # Only import to koji CG if the module is "done". diff --git a/module_build_service/builder/base.py b/module_build_service/builder/base.py index ecf36000..1fb9e1cb 100644 --- a/module_build_service/builder/base.py +++ b/module_build_service/builder/base.py @@ -337,9 +337,9 @@ class GenericBuilder(six.with_metaclass(ABCMeta)): raise NotImplementedError() @classmethod - def get_built_rpms_in_module_build(cls, build): + def get_built_rpms_in_module_build(cls, mmd): """ - :param ModuleBuild build: Module build to get the built RPMs from. + :param Modulemd mmd: Modulemd to get the built RPMs from. :return: list of NVRs """ raise NotImplementedError() diff --git a/module_build_service/resolver/DBResolver.py b/module_build_service/resolver/DBResolver.py index 6cf2581a..ad3dcaf9 100644 --- a/module_build_service/resolver/DBResolver.py +++ b/module_build_service/resolver/DBResolver.py @@ -22,8 +22,6 @@ # Written by Matt Prahl # Jan Kaluza -import kobo.rpmlib - from module_build_service import log from module_build_service.resolver.base import GenericResolver from module_build_service import models @@ -173,14 +171,13 @@ class DBResolver(GenericResolver): """ Resolves the requires list of N:S or N:S:V:C to a dictionary with keys as the module name and the values as a dictionary with keys of ref, - stream, version, filtered_rpms. + stream, version. If there are some modules loaded by utils.load_local_builds(...), these local modules will be considered when resolving the requires. A RuntimeError is raised on DB lookup errors. :param requires: a dictionary with the module name as the key and the stream as the value :return: a dictionary """ - from module_build_service.builder import GenericBuilder new_requires = {} with models.make_session(self.config) as session: for nsvc in requires: @@ -203,11 +200,7 @@ class DBResolver(GenericResolver): 'ref': None, 'stream': local_build.stream, 'version': local_build.version, - 'context': local_build.context, - # No need to set filtered_rpms for local builds, because MBS - # filters the RPMs automatically when the module build is - # done. - 'filtered_rpms': [] + 'context': local_build.context } continue @@ -222,7 +215,6 @@ class DBResolver(GenericResolver): raise UnprocessableEntity('The module {} was not found'.format(nsvc)) commit_hash = None - filtered_rpms = [] mmd = build.mmd() mbs_xmd = mmd.get_xmd().get('mbs') if mbs_xmd and 'commit' in mbs_xmd.keys(): @@ -237,22 +229,11 @@ class DBResolver(GenericResolver): 'The module "{}" is not built using Module Stream Expansion. ' 'Please rebuild this module first'.format(nsvc)) - # Find out the particular NVR of filtered packages - rpm_filter = mmd.get_rpm_filter() - if rpm_filter and rpm_filter.get(): - rpm_filter = rpm_filter.get() - built_nvrs = GenericBuilder.get_built_rpms_in_module_build(build) - for nvr in built_nvrs: - parsed_nvr = kobo.rpmlib.parse_nvr(nvr) - if parsed_nvr["name"] in rpm_filter: - filtered_rpms.append(nvr) - new_requires[module_name] = { 'ref': commit_hash, 'stream': module_stream, 'version': build.version, - 'context': build.context, - 'filtered_rpms': filtered_rpms, + 'context': build.context } return new_requires diff --git a/module_build_service/resolver/MBSResolver.py b/module_build_service/resolver/MBSResolver.py index 56a9b581..f4f23a38 100644 --- a/module_build_service/resolver/MBSResolver.py +++ b/module_build_service/resolver/MBSResolver.py @@ -266,8 +266,7 @@ class MBSResolver(GenericResolver): "module_name": { "ref": module_commit_hash, "stream": original_module_stream, - "version": module_version, - "filtered_rpms": ["nvr", ...] + "version": module_version }, ... } diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index ca2029f4..5ee98638 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -32,7 +32,8 @@ from module_build_service.utils import ( attempt_to_reuse_all_components, record_component_builds, get_rpm_release, - generate_koji_tag) + generate_koji_tag, + record_filtered_rpms) from module_build_service.errors import UnprocessableEntity, Forbidden, ValidationError from requests.exceptions import ConnectionError @@ -151,6 +152,7 @@ def init(config, session, msg): try: mmd = build.mmd() record_component_builds(mmd, build, session=session) + mmd = record_filtered_rpms(mmd) build.modulemd = mmd.dumps() build.transition(conf, models.BUILD_STATES["wait"]) # Catch custom exceptions that we can expose to the user diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index 71a19ee7..d4aaa1f0 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -29,6 +29,7 @@ import tempfile import os from multiprocessing.dummy import Pool as ThreadPool from datetime import datetime +import kobo.rpmlib from module_build_service import conf, db, log, models, Modulemd from module_build_service.errors import ( @@ -38,6 +39,53 @@ import module_build_service.scm from .mse import generate_expanded_mmds +def record_filtered_rpms(mmd): + """ + Reads the mmd["xmd"]["buildrequires"] and extends it with "filtered_rpms" + list containing the NVRs of filtered RPMs in a buildrequired module. + + :param Modulemd mmd: Modulemd of input module. + :rtype: Modulemd + :return: Modulemd extended with the "filtered_rpms" in XMD section. + """ + # Imported here to allow import of utils in GenericBuilder. + from module_build_service.builder import GenericBuilder + + new_buildrequires = {} + resolver = module_build_service.resolver.GenericResolver.create(conf) + for req_name, req_data in mmd.get_xmd()["mbs"]["buildrequires"].items(): + # In case this is module resubmit or local build, the filtered_rpms + # will already be there, so there is no point in generating them again. + if "filtered_rpms" in req_data: + continue + + # We can just get the first modulemd data from result right here thanks to + # strict=True, so in case the module cannot be found, get_module_modulemds + # raises an exception. + req_mmd = resolver.get_module_modulemds( + req_name, req_data["stream"], req_data["version"], req_data["context"], True)[0] + + # Find out the particular NVR of filtered packages + filtered_rpms = [] + rpm_filter = req_mmd.get_rpm_filter() + if rpm_filter and rpm_filter.get(): + rpm_filter = rpm_filter.get() + built_nvrs = GenericBuilder.backends[conf.system].get_built_rpms_in_module_build( + req_mmd) + for nvr in built_nvrs: + parsed_nvr = kobo.rpmlib.parse_nvr(nvr) + if parsed_nvr["name"] in rpm_filter: + filtered_rpms.append(nvr) + req_data["filtered_rpms"] = filtered_rpms + new_buildrequires[req_name] = req_data + + # Replace the old buildrequires with new ones. + xmd = glib.from_variant_dict(mmd.get_xmd()) + xmd["mbs"]["buildrequires"] = new_buildrequires + mmd.set_xmd(glib.dict_values(xmd)) + return mmd + + def _scm_get_latest(pkg): try: # If the modulemd specifies that the 'f25' branch is what diff --git a/tests/staged_data/includedmodules.yaml b/tests/staged_data/includedmodules.yaml index e0f79e25..8e4bdc4d 100644 --- a/tests/staged_data/includedmodules.yaml +++ b/tests/staged_data/includedmodules.yaml @@ -18,6 +18,9 @@ data: api: rpms: - bash + xmd: + mbs: + buildrequires: {} components: rpms: file: diff --git a/tests/staged_data/testmodule_init.yaml b/tests/staged_data/testmodule_init.yaml new file mode 100644 index 00000000..9ca7fe86 --- /dev/null +++ b/tests/staged_data/testmodule_init.yaml @@ -0,0 +1,45 @@ +document: modulemd +version: 1 +data: + summary: A test module in all its beautiful beauty + description: >- + This module demonstrates how to write simple modulemd files And + can be used for testing the build and release pipeline. + license: + module: [ MIT ] + dependencies: + buildrequires: + platform: f28 + requires: + platform: f28 + references: + community: https://docs.pagure.org/modularity/ + documentation: https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules + profiles: + default: + rpms: + - tangerine + xmd: + mbs: + buildrequires: + platform: + ref: virtual + stream: f28 + version: '3' + context: '00000000' + api: + rpms: + - perl-Tangerine + - tangerine + components: + rpms: + perl-List-Compare: + rationale: A dependency of tangerine. + ref: master + perl-Tangerine: + rationale: Provides API for this module and is a dependency of tangerine. + ref: master + tangerine: + rationale: Provides API for this module. + buildorder: 10 + ref: master diff --git a/tests/test_builder/test_koji.py b/tests/test_builder/test_koji.py index 92c87726..d32db3fb 100644 --- a/tests/test_builder/test_koji.py +++ b/tests/test_builder/test_koji.py @@ -32,7 +32,7 @@ import module_build_service.messaging import module_build_service.scheduler.handlers.repos import module_build_service.models import module_build_service.builder -from module_build_service import glib +from module_build_service import glib, db import pytest from mock import patch, MagicMock @@ -545,7 +545,17 @@ class TestKojiBuilder: 'size': 878684}], []) get_session.return_value = session - ret = KojiModuleBuilder.get_built_rpms_in_module_build(self.module) + # Module builds generated by init_data uses generic modulemd file and + # the module's name/stream/version/context does not have to match it. + # But for this test, we need it to match. + mmd = self.module.mmd() + self.module.name = mmd.get_name() + self.module.stream = mmd.get_stream() + self.module.version = mmd.get_version() + self.module.context = mmd.get_context() + db.session.commit() + + ret = KojiModuleBuilder.get_built_rpms_in_module_build(mmd) assert set(ret) == set( ['bar-2:1.30-4.el8+1308+551bfa71', 'tar-2:1.30-4.el8+1308+551bfa71']) diff --git a/tests/test_resolver/test_db.py b/tests/test_resolver/test_db.py index 05ca9d48..c4bbd6c9 100644 --- a/tests/test_resolver/test_db.py +++ b/tests/test_resolver/test_db.py @@ -120,25 +120,8 @@ class TestDBModule: ] assert set(result) == set(expected) - @patch("module_build_service.builder.base.GenericBuilder.get_built_rpms_in_module_build") - def test_resolve_requires(self, built_rpms): + def test_resolve_requires(self): build = models.ModuleBuild.query.get(2) - mmd = build.mmd() - filter_list = Modulemd.SimpleSet() - filter_list.add("foo") - filter_list.add("bar") - mmd.set_rpm_filter(filter_list) - build.modulemd = mmd.dumps() - db.session.commit() - - built_rpms.return_value = [ - "foo-0:2.4.48-3.el8+1308+551bfa71", - "foo-debuginfo-0:2.4.48-3.el8+1308+551bfa71", - "bar-0:2.5.48-3.el8+1308+551bfa71", - "bar-debuginfo-0:2.5.48-3.el8+1308+551bfa71", - "x-0:2.5.48-3.el8+1308+551bfa71", - "x-debuginfo-0:2.5.48-3.el8+1308+551bfa71"] - resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db') result = resolver.resolve_requires([":".join([ build.name, build.stream, build.version, build.context])]) @@ -146,10 +129,7 @@ class TestDBModule: assert result == { 'testmodule': { 'stream': 'master', 'version': '20170109091357', 'context': u'78e4a6fd', - 'ref': 'ff1ea79fc952143efeed1851aa0aa006559239ba', - 'filtered_rpms': [ - 'foo-0:2.4.48-3.el8+1308+551bfa71', - 'bar-0:2.5.48-3.el8+1308+551bfa71']}} + 'ref': 'ff1ea79fc952143efeed1851aa0aa006559239ba'}} def test_resolve_profiles(self): """ diff --git a/tests/test_scheduler/test_module_init.py b/tests/test_scheduler/test_module_init.py index 8bcd4d52..6380d358 100644 --- a/tests/test_scheduler/test_module_init.py +++ b/tests/test_scheduler/test_module_init.py @@ -28,7 +28,7 @@ from tests import conf, clean_database from tests.test_views.test_views import FakeSCM import module_build_service.messaging import module_build_service.scheduler.handlers.modules -from module_build_service import build_logs +from module_build_service import build_logs, Modulemd, db from module_build_service.models import make_session, ModuleBuild, ComponentBuild @@ -39,7 +39,7 @@ class TestModuleInit: self.staged_data_dir = os.path.join( os.path.dirname(__file__), '../', 'staged_data') testmodule_yml_path = os.path.join( - self.staged_data_dir, 'testmodule.yaml') + self.staged_data_dir, 'testmodule_init.yaml') with open(testmodule_yml_path, 'r') as f: yaml = f.read() scmurl = 'git://pkgs.domain.local/modules/testmodule?#620ec77' @@ -55,19 +55,43 @@ class TestModuleInit: except Exception: pass + @patch("module_build_service.builder.KojiModuleBuilder.KojiModuleBuilder." + "get_built_rpms_in_module_build") @patch('module_build_service.scm.SCM') - def test_init_basic(self, mocked_scm): - FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml', + def test_init_basic(self, mocked_scm, built_rpms): + FakeSCM(mocked_scm, 'testmodule', 'testmodule_init.yaml', '620ec77321b2ea7b0d67d82992dda3e1d67055b4') + + built_rpms.return_value = [ + "foo-0:2.4.48-3.el8+1308+551bfa71", + "foo-debuginfo-0:2.4.48-3.el8+1308+551bfa71", + "bar-0:2.5.48-3.el8+1308+551bfa71", + "bar-debuginfo-0:2.5.48-3.el8+1308+551bfa71", + "x-0:2.5.48-3.el8+1308+551bfa71", + "x-debuginfo-0:2.5.48-3.el8+1308+551bfa71"] + + platform_build = ModuleBuild.query.get(1) + mmd = platform_build.mmd() + filter_list = Modulemd.SimpleSet() + filter_list.add("foo") + filter_list.add("bar") + mmd.set_rpm_filter(filter_list) + platform_build.modulemd = mmd.dumps() + db.session.commit() + msg = module_build_service.messaging.MBSModule( msg_id=None, module_build_id=2, module_build_state='init') + with make_session(conf) as session: self.fn(config=conf, session=session, msg=msg) build = ModuleBuild.query.filter_by(id=2).one() # Make sure the module entered the wait state assert build.state == 1, build.state # Make sure format_mmd was run properly - assert type(build.mmd().get_xmd()['mbs']) is GLib.Variant + xmd_mbs = build.mmd().get_xmd()['mbs'] + assert type(xmd_mbs) is GLib.Variant + assert xmd_mbs["buildrequires"]["platform"]["filtered_rpms"] == [ + 'foo-0:2.4.48-3.el8+1308+551bfa71', 'bar-0:2.5.48-3.el8+1308+551bfa71'] @patch('module_build_service.scm.SCM') def test_init_scm_not_available(self, mocked_scm): @@ -90,7 +114,7 @@ class TestModuleInit: new_callable=PropertyMock, return_value=True) @patch('module_build_service.scm.SCM') def test_init_includedmodule(self, mocked_scm, mocked_mod_allow_repo): - FakeSCM(mocked_scm, "includedmodules", ['testmodule.yaml']) + FakeSCM(mocked_scm, "includedmodules", ['testmodule_init.yaml']) includedmodules_yml_path = os.path.join( self.staged_data_dir, 'includedmodules.yaml') with open(includedmodules_yml_path, 'r') as f: