From ab567803e8458b75f74dc1f0d37df49ff54f2f8c Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Mon, 19 Dec 2016 16:18:12 +0100 Subject: [PATCH] Fix #41 - Tag builds only in the end of the badge to not regen repository all the time. --- module_build_service/builder.py | 45 ++++++++++++++++-- module_build_service/config.py | 5 ++ module_build_service/scheduler/__init__.py | 3 +- .../scheduler/handlers/components.py | 42 ++++++++++------- tests/test_build/test_build.py | 46 +++++++++++++++++-- 5 files changed, 113 insertions(+), 28 deletions(-) diff --git a/module_build_service/builder.py b/module_build_service/builder.py index 992322a3..7d9a6a2b 100644 --- a/module_build_service/builder.py +++ b/module_build_service/builder.py @@ -212,6 +212,15 @@ class GenericBuilder(six.with_metaclass(ABCMeta)): """ raise NotImplementedError() + @abstractmethod + def tag_artifacts(self, artifacts): + """ + :param artifacts: list of artifacts (NVRs) to be tagged + + Adds the artifacts to tag associated with this module build. + """ + raise NotImplementedError() + @abstractmethod def build(self, artifact_name, source): """ @@ -282,6 +291,7 @@ class KojiModuleBuilder(GenericBuilder): """ self.owner = owner self.module_str = module + self.config = config self.tag_name = tag_name self.__prep = False log.debug("Using koji profile %r" % config.koji_profile) @@ -482,11 +492,11 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules This method is safe to call multiple times. """ log.info("%r adding artifacts %r" % (self, artifacts)) - dest_tag = self._get_tag(self.module_build_tag)['id'] + build_tag = self._get_tag(self.module_build_tag)['id'] for nvr in artifacts: - log.info("%r tagging %r into %r" % (self, nvr, dest_tag)) - self.koji_session.tagBuild(dest_tag, nvr, force=True) + log.info("%r tagging %r into %r" % (self, nvr, build_tag)) + self.koji_session.tagBuild(build_tag, nvr, force=True) if not install: continue @@ -494,7 +504,14 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules for group in ('srpm-build', 'build'): name = kobo.rpmlib.parse_nvr(nvr)['name'] log.info("%r adding %s to group %s" % (self, name, group)) - self.koji_session.groupPackageListAdd(dest_tag, group, name) + self.koji_session.groupPackageListAdd(build_tag, group, name) + + def tag_artifacts(self, artifacts): + dest_tag = self._get_tag(self.module_tag)['id'] + + for nvr in artifacts: + log.info("%r tagging %r into %r" % (self, nvr, dest_tag)) + self.koji_session.tagBuild(dest_tag, nvr, force=True) def wait_task(self, task_id): """ @@ -580,7 +597,19 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules self.koji_session.uploadWrapper(source, serverdir, callback=callback) source = "%s/%s" % (serverdir, os.path.basename(source)) - task_id = self.koji_session.build(source, self.module_target['name'], + # When "koji_build_macros_target" is set, we build the + # module-build-macros in this target instead of the self.module_target. + # The reason is that it is faster to build this RPM in + # already existing shared target, because Koji does not need to do + # repo-regen. + if (artifact_name == "module-build-macros" + and self.config.koji_build_macros_target): + module_target = self.config.koji_build_macros_target + else: + module_target = self.module_target['name'] + + build_opts = {"skip_tag": True} + task_id = self.koji_session.build(source, module_target, build_opts, priority=self.build_priority) log.info("submitted build of %s (task_id=%s), via %s" % ( source, task_id, self)) @@ -874,6 +903,9 @@ class CoprModuleBuilder(GenericBuilder): if backend == "copr": return self._dependency_repo(module, arch, "koji") + def tag_artifacts(self, artifacts): + pass + def build(self, artifact_name, source): """ :param artifact_name : A package name. We can't guess it since macros @@ -1153,6 +1185,9 @@ $repos _execute_cmd(["mock", "-r", self.mock_config, "-i", "module-build-macros"]) + def tag_artifacts(self, artifacts): + pass + def buildroot_add_repos(self, dependencies): # TODO: We support only dependencies from Koji here. This should be # extended to Copr in the future. diff --git a/module_build_service/config.py b/module_build_service/config.py index 35bbef6e..9361adbc 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -116,6 +116,11 @@ class Config(object): 'type': str, 'default': None, 'desc': 'Koji repository URL.'}, + 'koji_build_macros_target': { + 'type': str, + 'default': '', + 'desc': 'Target to build "module-build-macros" RPM in.' + }, 'rpms_default_repository': { 'type': str, 'default': 'git://pkgs.fedoraproject.org/rpms/', diff --git a/module_build_service/scheduler/__init__.py b/module_build_service/scheduler/__init__.py index ccde5b70..3c81d6bf 100644 --- a/module_build_service/scheduler/__init__.py +++ b/module_build_service/scheduler/__init__.py @@ -9,7 +9,6 @@ import module_build_service.scheduler.consumer import logging log = logging.getLogger(__name__) - def main(initial_messages, stop_condition): """ Run the consumer until some condition is met. @@ -36,6 +35,8 @@ def main(initial_messages, stop_condition): options=config, # Only run the specified consumers if any are so specified. consumers=consumers, + # Do not run default producers. + producers=[], # Tell moksha to quiet its logging. framework=False, ) diff --git a/module_build_service/scheduler/handlers/components.py b/module_build_service/scheduler/handlers/components.py index 85309116..6ed790d1 100644 --- a/module_build_service/scheduler/handlers/components.py +++ b/module_build_service/scheduler/handlers/components.py @@ -38,7 +38,7 @@ logging.basicConfig(level=logging.DEBUG) def _finalize(config, session, msg, state): """ Called whenever a koji build completes or fails. """ - # First, find our ModuleBuild associated with this repo, if any. + # First, find our ModuleBuild associated with this component, if any. component_build = models.ComponentBuild.from_component_event(session, msg) try: nvr = "{}-{}-{}".format(msg.build_name, msg.build_version, @@ -63,18 +63,26 @@ def _finalize(config, session, msg, state): parent = component_build.module_build - if component_build.package == 'module-build-macros': - if state != koji.BUILD_STATES['COMPLETE']: - # If the macro build failed, then the module is doomed. - parent.transition(config, state=models.BUILD_STATES['failed'], - state_reason=state_reason) - session.commit() - return + # If the macro build failed, then the module is doomed. + if (component_build.package == 'module-build-macros' + and state != koji.BUILD_STATES['COMPLETE']): + parent.transition(config, state=models.BUILD_STATES['failed'], + state_reason=state_reason) + session.commit() + return + + # If there are no other components still building in a batch, + # we can tag all successfully built components in the batch. + unbuilt_components_in_batch = [ + c for c in parent.current_batch() + if c.state == koji.BUILD_STATES['BUILDING'] or not c.state + ] + if not unbuilt_components_in_batch: + built_components_in_batch = [ + c.nvr for c in parent.current_batch() + if c.state == koji.BUILD_STATES['COMPLETE'] + ] - # TODO -- we should just do this with a koji target that feeds -build. - # Otherwise.. if it didn't fail, then tag it. - if state == koji.BUILD_STATES['COMPLETE']: - # And install the macros. module_name = parent.name tag = parent.koji_tag builder = module_build_service.builder.GenericBuilder.create( @@ -88,16 +96,16 @@ def _finalize(config, session, msg, state): except ValueError: reason = "Failed to gather buildroot groups from SCM." log.exception(reason) - parent.transition(config, state="failed", state_reason=reason) + parent.transition(config, state=models.BUILD_STATES["failed"], + state_reason=reason) session.commit() raise builder.buildroot_connect(groups) - # tag && add to srpm-build group - nvr = "{}-{}-{}".format(msg.build_name, msg.build_version, - msg.build_release) + # tag && add to srpm-build group if neccessary install = bool(component_build.package == 'module-build-macros') - builder.buildroot_add_artifacts([nvr,], install=install) + builder.buildroot_add_artifacts(built_components_in_batch, install=install) + builder.tag_artifacts(built_components_in_batch) session.commit() diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index f6aff7b5..1e3e35d7 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -78,6 +78,8 @@ class TestModuleBuilder(GenericBuilder): on_build_cb = None on_cancel_cb = None + on_buildroot_add_artifacts_cb = None + on_tag_artifacts_cb = None def __init__(self, owner, module, config, tag_name): self.module_str = module @@ -90,6 +92,8 @@ class TestModuleBuilder(GenericBuilder): TestModuleBuilder.INSTANT_COMPLETE = False TestModuleBuilder.on_build_cb = None TestModuleBuilder.on_cancel_cb = None + TestModuleBuilder.on_buildroot_add_artifacts_cb = None + TestModuleBuilder.on_tag_artifacts_cb = None def buildroot_connect(self, groups): pass @@ -107,11 +111,16 @@ class TestModuleBuilder(GenericBuilder): pass def buildroot_add_artifacts(self, artifacts, install=False): - pass + if TestModuleBuilder.on_buildroot_add_artifacts_cb: + TestModuleBuilder.on_buildroot_add_artifacts_cb(self, artifacts, install) def buildroot_add_repos(self, dependencies): pass + def tag_artifacts(self, artifacts): + if TestModuleBuilder.on_tag_artifacts_cb: + TestModuleBuilder.on_tag_artifacts_cb(self, artifacts) + @property def module_build_tag(self): return {"name": self.tag_name + "-build"} @@ -130,7 +139,7 @@ class TestModuleBuilder(GenericBuilder): msg_id='a faked internal message', build_id=build_id, task_id=build_id, - build_name="name", + build_name=path.basename(source), build_new_state=state, build_release="1", build_version="1" @@ -211,7 +220,30 @@ class TestBuild(unittest.TestCase): data = json.loads(rv.data) module_build_id = data['id'] - msgs = [RidaModule("fake msg", 1, 1)] + # Check that components are tagged after the batch is built. + tag_groups = [] + tag_groups.append([u'module-build-macros-0.1-1.module_testmodule_teststream_1.src.rpm-1-1']) + tag_groups.append([u'perl-Tangerine?#f25-1-1', u'perl-List-Compare?#f25-1-1']) + tag_groups.append([u'tangerine?#f25-1-1']) + + def on_tag_artifacts_cb(cls, artifacts): + self.assertEqual(tag_groups.pop(0), artifacts) + + TestModuleBuilder.on_tag_artifacts_cb = on_tag_artifacts_cb + + # Check that the components are added to buildroot after the batch + # is built. + buildroot_groups = [] + buildroot_groups.append([u'module-build-macros-0.1-1.module_testmodule_teststream_1.src.rpm-1-1']) + buildroot_groups.append([u'perl-Tangerine?#f25-1-1', u'perl-List-Compare?#f25-1-1']) + buildroot_groups.append([u'tangerine?#f25-1-1']) + + def on_buildroot_add_artifacts_cb(cls, artifacts, install): + self.assertEqual(buildroot_groups.pop(0), artifacts) + + TestModuleBuilder.on_buildroot_add_artifacts_cb = on_buildroot_add_artifacts_cb + + msgs = [] stop = module_build_service.scheduler.make_simple_stop_condition(db.session) module_build_service.scheduler.main(msgs, stop) @@ -221,6 +253,10 @@ class TestBuild(unittest.TestCase): self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE']) self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] ) + # All components has to be tagged, so tag_groups and buildroot_groups are empty... + self.assertEqual(tag_groups, []) + self.assertEqual(buildroot_groups, []) + @timed(30) @patch('module_build_service.auth.get_username', return_value='Homer J. Simpson') @patch('module_build_service.auth.assert_is_packager') @@ -257,7 +293,7 @@ class TestBuild(unittest.TestCase): TestModuleBuilder.on_build_cb = on_build_cb TestModuleBuilder.on_cancel_cb = on_cancel_cb - msgs = [RidaModule("fake msg", 1, 1)] + msgs = [] stop = module_build_service.scheduler.make_simple_stop_condition(db.session) module_build_service.scheduler.main(msgs, stop) @@ -296,7 +332,7 @@ class TestBuild(unittest.TestCase): TestModuleBuilder.BUILD_STATE = "BUILDING" TestModuleBuilder.INSTANT_COMPLETE = True - msgs = [RidaModule("fake msg", 1, 1)] + msgs = [] stop = module_build_service.scheduler.make_simple_stop_condition(db.session) module_build_service.scheduler.main(msgs, stop)