From 3de12fee23da9d67c5c2f2f9d458f5de4f8eb2bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Tue, 14 Mar 2017 11:19:45 +0100 Subject: [PATCH 1/7] Create record about module and build it separately --- module_build_service/builder.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/module_build_service/builder.py b/module_build_service/builder.py index 80d99b45..8df80852 100644 --- a/module_build_service/builder.py +++ b/module_build_service/builder.py @@ -978,6 +978,7 @@ class CoprModuleBuilder(GenericBuilder): Koji Example: create tag, targets, set build tag inheritance... """ self.copr = self._get_copr_safe() + self._create_module_safe() if self.copr and self.copr.projectname and self.copr.username: self.__prep = True log.info("%r buildroot sucessfully connected." % self) @@ -1006,6 +1007,27 @@ class CoprModuleBuilder(GenericBuilder): # @TODO fix issues with custom-1-x86_64 and custom-1-i386 chroot and use it return self.client.create_project(ownername, projectname, ["fedora-24-x86_64"]) + def _create_module_safe(self): + from copr.exceptions import CoprRequestException + + # @TODO it would be nice if the module build object was passed to Builder __init__ + module = ModuleBuild.query.filter(ModuleBuild.name == self.module_str).one() + modulemd = tempfile.mktemp() + module.mmd().dump(modulemd) + + kwargs = { + "username": module.copr_owner or self.owner, + "projectname": module.copr_project or CoprModuleBuilder._tag_to_copr_name(self.tag_name), + "modulemd": modulemd, + "create": True, + "build": False, + } + try: + self.client.make_module(**kwargs) + except CoprRequestException as ex: + if "already exists" not in ex.message.get("nsv", [""])[0]: + raise RuntimeError("Buildroot is not prep-ed") + def buildroot_ready(self, artifacts=None): """ :param artifacts=None : a list of artifacts supposed to be in the buildroot @@ -1123,8 +1145,8 @@ class CoprModuleBuilder(GenericBuilder): return log.info("Missing builds, not going to create a module") # Create a module from previous project - kwargs = {"username": self.copr.username, "projectname": self.copr.projectname, "modulemd": modulemd} - result = self.client.create_new_build_module(**kwargs) + result = self.client.make_module(username=self.copr.username, projectname=self.copr.projectname, + modulemd=modulemd, create=False, build=True) if result.output != "ok": log.error(result.error) return From 61b74293b1af7d045b2f15683d4d9817c3261a83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Tue, 21 Mar 2017 07:47:25 +0100 Subject: [PATCH 2/7] Rather extend koji message so we can have additional arguments for copr --- module_build_service/messaging.py | 10 ++++++---- module_build_service/models.py | 2 +- module_build_service/scheduler/consumer.py | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/module_build_service/messaging.py b/module_build_service/messaging.py index b7413ec5..267ecdd4 100644 --- a/module_build_service/messaging.py +++ b/module_build_service/messaging.py @@ -216,11 +216,12 @@ class BaseMessage(object): msg_id, msg_inner_msg.get('id'), msg_inner_msg.get('state')) elif conf.system == category == 'copr' and object == 'build': + copr = msg_inner_msg.get('copr') build = msg_inner_msg.get('build') status = msg_inner_msg.get('status') pkg = msg_inner_msg.get('pkg') what = msg_inner_msg.get('what') - msg_obj = CoprBuildEnd(msg_id, build, status, pkg, what) + msg_obj = CoprBuildEnd(msg_id, build, status, copr, pkg, what) # If the message matched the regex and is important to the app, # it will be returned @@ -271,7 +272,7 @@ class KojiRepoChange(BaseMessage): self.repo_tag = repo_tag -class CoprBuildEnd(object): +class CoprBuildEnd(KojiBuildChange): """ A wrapper class that transforms copr message attributes to a KojiBuildChange message object :param msg_id: the id of the msg (e.g. 2016-SomeGUID) @@ -282,9 +283,9 @@ class CoprBuildEnd(object): (e.g. mutt-kz-1.5.23.1-1.20150203.git.c8504a8a.fc21) :param state_reason: the optional reason as to why the state changed """ - def __new__(cls, msg_id, build_id, status, pkg, what=None): + def __init__(self, msg_id, build_id, status, copr, pkg, what=None): nvr = kobo.rpmlib.parse_nvra(pkg) - return KojiBuildChange( + super(CoprBuildEnd, self).__init__( msg_id=msg_id, build_id=build_id, task_id=build_id, @@ -294,6 +295,7 @@ class CoprBuildEnd(object): build_release=".".join(s for s in [nvr["release"], nvr["epoch"], nvr["arch"]] if s), state_reason=what, ) + self.copr = copr class MBSModule(BaseMessage): diff --git a/module_build_service/models.py b/module_build_service/models.py index b7542560..c0d25a0b 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -375,7 +375,7 @@ class ComponentBuild(MBSBase): @classmethod def from_component_event(cls, session, event): - if type(event) == module_build_service.messaging.KojiBuildChange: + if isinstance(event, module_build_service.messaging.KojiBuildChange): if event.module_build_id: return session.query(cls).filter_by( task_id=event.task_id, module_id=event.module_build_id)\ diff --git a/module_build_service/scheduler/consumer.py b/module_build_service/scheduler/consumer.py index c9c66f28..dd824a8a 100644 --- a/module_build_service/scheduler/consumer.py +++ b/module_build_service/scheduler/consumer.py @@ -178,7 +178,7 @@ class MBSConsumer(fedmsg.consumers.FedmsgConsumer): build = None # Choose a handler for this message - if type(msg) == module_build_service.messaging.KojiBuildChange: + if isinstance(msg, module_build_service.messaging.KojiBuildChange): handler = self.on_build_change[msg.build_new_state] build = models.ComponentBuild.from_component_event(session, msg) if build: From 6ec3a63517ce3cdb2672f572b51c37ac91f7ded6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Tue, 21 Mar 2017 07:50:54 +0100 Subject: [PATCH 3/7] Remove waiting till all builds are finished (old fedmsg workaround) --- module_build_service/builder.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/module_build_service/builder.py b/module_build_service/builder.py index 8df80852..a15ab01a 100644 --- a/module_build_service/builder.py +++ b/module_build_service/builder.py @@ -1118,32 +1118,11 @@ class CoprModuleBuilder(GenericBuilder): return response.data["ids"][0], koji.BUILD_STATES["BUILDING"], response.message, None - def _wait_until_all_builds_are_finished(self, module): - while True: - states = {b: self.client.get_build_details(b.task_id).status for b in module.component_builds} - if "failed" in states.values(): - raise ValueError("Some builds failed") - - if not filter(lambda x: x != "succeeded", states.values()): - return - - seconds = 60 - log.info("Going to sleep for {}s to wait until builds in copr are finished".format(seconds)) - time.sleep(seconds) - def finalize(self): modulemd = tempfile.mktemp() m1 = ModuleBuild.query.filter(ModuleBuild.name == self.module_str).one() m1.mmd().dump(modulemd) - # Wait until all builds are finished - # We shouldn't do this once the fedmsg on copr is done - from copr.exceptions import CoprRequestException - try: - self._wait_until_all_builds_are_finished(m1) - except (CoprRequestException, ValueError): - return log.info("Missing builds, not going to create a module") - # Create a module from previous project result = self.client.make_module(username=self.copr.username, projectname=self.copr.projectname, modulemd=modulemd, create=False, build=True) From a24b49e162ba2d66e22ea7b01d190cd59042e88f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Tue, 21 Mar 2017 07:59:35 +0100 Subject: [PATCH 4/7] Generate buildsys.repo.done locally --- module_build_service/messaging.py | 12 ++++++++++++ .../scheduler/handlers/components.py | 9 ++++++++- tests/test_messaging.py | 9 ++++++++- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/module_build_service/messaging.py b/module_build_service/messaging.py index 267ecdd4..0d163cb2 100644 --- a/module_build_service/messaging.py +++ b/module_build_service/messaging.py @@ -298,6 +298,18 @@ class CoprBuildEnd(KojiBuildChange): self.copr = copr +class CoprRepoDone(object): + """There is actually no repo.done message in Copr + This is a class for constructing buildsys.repo.done + and triggering it locally""" + def __init__(self, copr): + self.copr = copr + + def publish(self): + msg = {"tag": "{}-build".format(self.copr)} + publish("repo.done", msg, conf, "buildsys") + + class MBSModule(BaseMessage): """ A class that inherits from BaseMessage to provide a message object for a module event generated by module_build_service diff --git a/module_build_service/scheduler/handlers/components.py b/module_build_service/scheduler/handlers/components.py index 52665043..a04a69ce 100644 --- a/module_build_service/scheduler/handlers/components.py +++ b/module_build_service/scheduler/handlers/components.py @@ -30,7 +30,7 @@ import module_build_service.pdc import koji -from module_build_service import models, log +from module_build_service import models, log, messaging logging.basicConfig(level=logging.DEBUG) @@ -111,6 +111,13 @@ def _finalize(config, session, msg, state): builder.tag_artifacts(built_components_in_batch) session.commit() + + # Start of new batch is triggered by buildys.repo.done message. + # However in Copr there is no such thing. Therefore, + # since the batch is done, we can state that repo is done + if config.system == "copr": + messaging.CoprRepoDone(msg.copr).publish() + elif (any([c.state != koji.BUILD_STATES['BUILDING'] for c in unbuilt_components_in_batch])): # We are not in the middle of the batch building and diff --git a/tests/test_messaging.py b/tests/test_messaging.py index 97043f55..c13a3132 100644 --- a/tests/test_messaging.py +++ b/tests/test_messaging.py @@ -23,7 +23,7 @@ import unittest from module_build_service import messaging, conf -from mock import patch, PropertyMock +from mock import patch, PropertyMock, ANY class TestFedmsgMessaging(unittest.TestCase): @@ -90,3 +90,10 @@ class TestFedmsgMessaging(unittest.TestCase): self.assertEqual(msg.build_release, '1.20150203.git.c8504a8a.fc21') self.assertEqual(msg.state_reason, 'build end: user:fatka copr:mutt-kz build:100 ip:172.16.3.3 pid:12010 status:1') + + @patch("module_build_service.messaging.publish") + def test_copr_repo_done(self, publish): + messaging.CoprRepoDone('someprojectname').publish() + self.assertTrue(publish.called) + repo_change_msg = {'tag': 'someprojectname-build'} + publish.assert_called_with("repo.done", repo_change_msg, ANY, "buildsys") From ffbec05c6321e44e859f6091458de155b5678ea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Thu, 23 Mar 2017 19:56:00 +0100 Subject: [PATCH 5/7] Update CoprBuildEnd docstring --- module_build_service/messaging.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/module_build_service/messaging.py b/module_build_service/messaging.py index 0d163cb2..b4d2eb36 100644 --- a/module_build_service/messaging.py +++ b/module_build_service/messaging.py @@ -273,12 +273,18 @@ class KojiRepoChange(BaseMessage): class CoprBuildEnd(KojiBuildChange): - """ A wrapper class that transforms copr message attributes - to a KojiBuildChange message object + """ A class that inherits from KojiBuildChange to provide a message + object for a build info from Copr + + @TODO There should be a base class for CoprBuildEnd and KojiBuildChange + and conditions in the code should check for it's descendants instead of KojiBuildChange directly. + In such case this class would not have to inherit from koji class + :param msg_id: the id of the msg (e.g. 2016-SomeGUID) :param build_id: the id of the build (e.g. 264382) :param status: the new build state (see http://copr-backend.readthedocs.io/package/constants.html#backend.constants.BuildStatus ) + :param copr: the project name :param pkg: the full name of what is being built (e.g. mutt-kz-1.5.23.1-1.20150203.git.c8504a8a.fc21) :param state_reason: the optional reason as to why the state changed From e00797873ee48541ac880ae6fec87d6110a29032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Thu, 23 Mar 2017 20:02:31 +0100 Subject: [PATCH 6/7] Remove the dumped modulemd file after requests --- module_build_service/builder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/module_build_service/builder.py b/module_build_service/builder.py index a15ab01a..15edaef2 100644 --- a/module_build_service/builder.py +++ b/module_build_service/builder.py @@ -1027,6 +1027,8 @@ class CoprModuleBuilder(GenericBuilder): except CoprRequestException as ex: if "already exists" not in ex.message.get("nsv", [""])[0]: raise RuntimeError("Buildroot is not prep-ed") + finally: + os.remove(modulemd) def buildroot_ready(self, artifacts=None): """ @@ -1126,6 +1128,7 @@ class CoprModuleBuilder(GenericBuilder): # Create a module from previous project result = self.client.make_module(username=self.copr.username, projectname=self.copr.projectname, modulemd=modulemd, create=False, build=True) + os.remove(modulemd) if result.output != "ok": log.error(result.error) return From 70b3782231af612bfd8d0b57fc0beec12bd43c70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Tue, 28 Mar 2017 19:13:38 +0200 Subject: [PATCH 7/7] Fake repo.done message without publishing it --- module_build_service/builder.py | 13 ++++++++++++- module_build_service/messaging.py | 12 ------------ .../scheduler/handlers/components.py | 9 +-------- tests/test_messaging.py | 9 +-------- 4 files changed, 14 insertions(+), 29 deletions(-) diff --git a/module_build_service/builder.py b/module_build_service/builder.py index 15edaef2..0e0b5285 100644 --- a/module_build_service/builder.py +++ b/module_build_service/builder.py @@ -1058,7 +1058,18 @@ class CoprModuleBuilder(GenericBuilder): # This forces install of bash into buildroot and srpm-buildroot koji add-group-pkg $module-build-tag srpm-build bash """ - pass + + # Start of a new batch of builds is triggered by buildsys.repo.done message. + # However in Copr there is no such thing. Therefore we are going to fake + # the message when builds are finished + self._send_repo_done() + + def _send_repo_done(self): + msg = module_build_service.messaging.KojiRepoChange( + msg_id='a faked internal message', + repo_tag=self.tag_name + "-build", + ) + module_build_service.scheduler.consumer.work_queue_put(msg) def buildroot_add_repos(self, dependencies): log.info("%r adding deps on %r" % (self, dependencies)) diff --git a/module_build_service/messaging.py b/module_build_service/messaging.py index b4d2eb36..cfc29880 100644 --- a/module_build_service/messaging.py +++ b/module_build_service/messaging.py @@ -304,18 +304,6 @@ class CoprBuildEnd(KojiBuildChange): self.copr = copr -class CoprRepoDone(object): - """There is actually no repo.done message in Copr - This is a class for constructing buildsys.repo.done - and triggering it locally""" - def __init__(self, copr): - self.copr = copr - - def publish(self): - msg = {"tag": "{}-build".format(self.copr)} - publish("repo.done", msg, conf, "buildsys") - - class MBSModule(BaseMessage): """ A class that inherits from BaseMessage to provide a message object for a module event generated by module_build_service diff --git a/module_build_service/scheduler/handlers/components.py b/module_build_service/scheduler/handlers/components.py index a04a69ce..52665043 100644 --- a/module_build_service/scheduler/handlers/components.py +++ b/module_build_service/scheduler/handlers/components.py @@ -30,7 +30,7 @@ import module_build_service.pdc import koji -from module_build_service import models, log, messaging +from module_build_service import models, log logging.basicConfig(level=logging.DEBUG) @@ -111,13 +111,6 @@ def _finalize(config, session, msg, state): builder.tag_artifacts(built_components_in_batch) session.commit() - - # Start of new batch is triggered by buildys.repo.done message. - # However in Copr there is no such thing. Therefore, - # since the batch is done, we can state that repo is done - if config.system == "copr": - messaging.CoprRepoDone(msg.copr).publish() - elif (any([c.state != koji.BUILD_STATES['BUILDING'] for c in unbuilt_components_in_batch])): # We are not in the middle of the batch building and diff --git a/tests/test_messaging.py b/tests/test_messaging.py index c13a3132..97043f55 100644 --- a/tests/test_messaging.py +++ b/tests/test_messaging.py @@ -23,7 +23,7 @@ import unittest from module_build_service import messaging, conf -from mock import patch, PropertyMock, ANY +from mock import patch, PropertyMock class TestFedmsgMessaging(unittest.TestCase): @@ -90,10 +90,3 @@ class TestFedmsgMessaging(unittest.TestCase): self.assertEqual(msg.build_release, '1.20150203.git.c8504a8a.fc21') self.assertEqual(msg.state_reason, 'build end: user:fatka copr:mutt-kz build:100 ip:172.16.3.3 pid:12010 status:1') - - @patch("module_build_service.messaging.publish") - def test_copr_repo_done(self, publish): - messaging.CoprRepoDone('someprojectname').publish() - self.assertTrue(publish.called) - repo_change_msg = {'tag': 'someprojectname-build'} - publish.assert_called_with("repo.done", repo_change_msg, ANY, "buildsys")