From 1eb3c653dd7fd68e2783131fce24861be9cc59ed Mon Sep 17 00:00:00 2001 From: Martin Curlej Date: Mon, 30 Oct 2017 19:37:23 +0100 Subject: [PATCH] Changed the filters so they execute when everything is build Signed-off-by: Martin Curlej --- .../builder/MockModuleBuilder.py | 9 +- module_build_service/models.py | 4 + .../staged_data/testmodule-with-filters.yaml | 31 ++++ tests/test_builder/test_mock.py | 147 ++++++++++++++++++ 4 files changed, 187 insertions(+), 4 deletions(-) create mode 100644 tests/staged_data/testmodule-with-filters.yaml create mode 100644 tests/test_builder/test_mock.py diff --git a/module_build_service/builder/MockModuleBuilder.py b/module_build_service/builder/MockModuleBuilder.py index d122238e..4e8366b0 100644 --- a/module_build_service/builder/MockModuleBuilder.py +++ b/module_build_service/builder/MockModuleBuilder.py @@ -178,10 +178,11 @@ class MockModuleBuilder(GenericBuilder): if not rpm.endswith(".rpm"): continue - # If RPM is filtered-out, do not add it to artifacts list. - nvr = kobo.rpmlib.parse_nvr(rpm) - if nvr["name"] in m1_mmd.filter.rpms: - continue + if m1.last_batch_id() == m1.batch: + # If RPM is filtered-out, do not add it to artifacts list. + nvr = kobo.rpmlib.parse_nvr(rpm) + if nvr["name"] in m1_mmd.filter.rpms: + continue pkglist_f.write(rpm + '\n') rpm = rpm[:-len(".rpm")] diff --git a/module_build_service/models.py b/module_build_service/models.py index e6f130da..5b560e31 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -181,6 +181,10 @@ class ModuleBuild(MBSBase): if component.batch == self.batch ] + def last_batch_id(self): + """ Returns the id of the last batch """ + return max([build.batch for build in self.component_builds]) + def up_to_current_batch(self, state=None): """ Returns all components of this module in the current batch and diff --git a/tests/staged_data/testmodule-with-filters.yaml b/tests/staged_data/testmodule-with-filters.yaml new file mode 100644 index 00000000..f59e1523 --- /dev/null +++ b/tests/staged_data/testmodule-with-filters.yaml @@ -0,0 +1,31 @@ +document: modulemd +version: 1 +data: + summary: A test module in all its beautiful beauty + description: >- + This module demonstrates how to write simple modulemd files And + can be used for testing the build and release pipeline. + license: + module: [ MIT ] + dependencies: + buildrequires: + platform: master + # Buildrequire Host until MBS implements transitive deps + host: master + requires: + platform: master + references: + community: https://docs.pagure.org/modularity/ + documentation: https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules + filter: + rpms: + - ed + components: + rpms: + ed: + rationale: A build dependency of mksh, to test buildorder. + ref: master + mksh: + rationale: The main component of this module. + ref: master + buildorder: 1 diff --git a/tests/test_builder/test_mock.py b/tests/test_builder/test_mock.py new file mode 100644 index 00000000..8ee55ab0 --- /dev/null +++ b/tests/test_builder/test_mock.py @@ -0,0 +1,147 @@ +import os +import unittest +import mock +import koji +import tempfile +import shutil + +import kobo.rpmlib + +from module_build_service import conf +from module_build_service.models import ModuleBuild, ComponentBuild, make_session +from module_build_service.builder import MockModuleBuilder +from tests import db, init_data + + +class TestMockModuleBuilder(unittest.TestCase): + + def setUp(self): + db.drop_all() + db.create_all() + self.resultdir = tempfile.mkdtemp() + + def tearDown(self): + init_data() + shutil.rmtree(self.resultdir) + + def _create_module_with_filters(self, session, batch, state): + comp_builds = [ + { + "module_id": 1, + "package": "ed", + "format": "rpms", + "scmurl": ("git://pkgs.fedoraproject.org/rpms/ed" + "?#01bf8330812fea798671925cc537f2f29b0bd216"), + "batch": 2, + "ref": "01bf8330812fea798671925cc537f2f29b0bd216" + }, + { + "module_id": 1, + "package": "mksh", + "format": "rpms", + "scmurl": ("git://pkgs.fedoraproject.org/rpms/mksh" + "?#f70fd11ddf96bce0e2c64309706c29156b39141d"), + "batch": 3, + "ref": "f70fd11ddf96bce0e2c64309706c29156b39141d" + }, + ] + + base_dir = os.path.abspath(os.path.dirname(__file__)) + modulemd_path = os.path.join( + base_dir, '..', 'staged_data', 'testmodule-with-filters.yaml') + + with open(modulemd_path, "r") as fd: + module = ModuleBuild.create( + session, + conf, + name="mbs-testmodule", + stream="test", + version="20171027111452", + modulemd=fd.read(), + scmurl="file:///testdir", + username="test", + ) + module.koji_tag = "module-mbs-testmodule-test-20171027111452" + md = module.mmd() + md.xmd = { + 'mbs': { + 'rpms': { + 'ed': {'ref': '01bf8330812fea798671925cc537f2f29b0bd216'}, + 'mksh': {'ref': 'f70fd11ddf96bce0e2c64309706c29156b39141d'} + }, + 'buildrequires': + { + 'host': { + 'version': '20171024133034', + 'filtered_rpms': [], + 'stream': 'master', + 'ref': '6df253bb3c53e84706c01b8ab2d5cac24f0b6d45' + }, + 'platform': { + 'version': '20171028112959', + 'filtered_rpms': [], + 'stream': 'master', + 'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa'} + }, + 'scmurl': 'file:///testdir', + 'commit': '5566bc792ec7a03bb0e28edd1b104a96ba342bd8', + 'requires': { + 'platform': { + 'version': '20171028112959', + 'filtered_rpms': [], + 'stream': 'master', + 'ref': '4f7787370a931d57421f9f9555fc41c3e31ff1fa'} + } + } + } + module.modulemd = md.dumps() + module.batch = batch + session.add(module) + + for build in comp_builds: + cb = ComponentBuild(**dict(build, format="rpms", state=state)) + session.add(cb) + session.commit() + + return module + + @mock.patch("module_build_service.conf.system", new="mock") + def test_createrepo_filter_last_batch(self, *args): + with make_session(conf) as session: + module = self._create_module_with_filters(session, 3, koji.BUILD_STATES['COMPLETE']) + + builder = MockModuleBuilder("mcurlej", module, conf, module.koji_tag, + module.component_builds) + builder.resultsdir = self.resultdir + rpms = [ + "ed-1.14.1-4.module+24957a32.x86_64.rpm", + "mksh-56b-1.module+24957a32.x86_64.rpm", + "module-build-macros-0.1-1.module+24957a32.noarch.rpm" + ] + with mock.patch("os.listdir", return_value=rpms): + builder._createrepo() + + with open(os.path.join(self.resultdir, "pkglist"), "r") as fd: + pkglist = fd.read().strip() + rpm_names = [kobo.rpmlib.parse_nvr(rpm)["name"] for rpm in pkglist.split('\n')] + assert "ed" not in rpm_names + + @mock.patch("module_build_service.conf.system", new="mock") + def test_createrepo_not_last_batch(self): + with make_session(conf) as session: + module = self._create_module_with_filters(session, 2, koji.BUILD_STATES['COMPLETE']) + + builder = MockModuleBuilder("mcurlej", module, conf, module.koji_tag, + module.component_builds) + builder.resultsdir = self.resultdir + rpms = [ + "ed-1.14.1-4.module+24957a32.x86_64.rpm", + "mksh-56b-1.module+24957a32.x86_64.rpm", + ] + with mock.patch("os.listdir", return_value=rpms): + builder._createrepo() + + with open(os.path.join(self.resultdir, "pkglist"), "r") as fd: + pkglist = fd.read().strip() + rpm_names = [kobo.rpmlib.parse_nvr(rpm)["name"] for rpm in pkglist.split('\n')] + assert "ed" in rpm_names