Merge #448 Copr builder improvements #2

module_build_service/builder.py

@@ -981,6 +981,7 @@ class CoprModuleBuilder(GenericBuilder):
         Koji Example: create tag, targets, set build tag inheritance...
         """
         self.copr = self._get_copr_safe()
+        self._create_module_safe()
         if self.copr and self.copr.projectname and self.copr.username:
             self.__prep = True
         log.info("%r buildroot sucessfully connected." % self)
@@ -1009,6 +1010,29 @@ class CoprModuleBuilder(GenericBuilder):
         # @TODO fix issues with custom-1-x86_64 and custom-1-i386 chroot and use it
         return self.client.create_project(ownername, projectname, ["fedora-24-x86_64"])
 
+    def _create_module_safe(self):
+        from copr.exceptions import CoprRequestException
+
+        # @TODO it would be nice if the module build object was passed to Builder __init__
+        module = ModuleBuild.query.filter(ModuleBuild.name == self.module_str).one()
+        modulemd = tempfile.mktemp()
+        module.mmd().dump(modulemd)
+
+        kwargs = {
+            "username": module.copr_owner or self.owner,
+            "projectname": module.copr_project or CoprModuleBuilder._tag_to_copr_name(self.tag_name),
+            "modulemd": modulemd,
+            "create": True,
+            "build": False,
+        }
+        try:
+            self.client.make_module(**kwargs)
+        except CoprRequestException as ex:
+            if "already exists" not in ex.message.get("nsv", [""])[0]:
+                raise RuntimeError("Buildroot is not prep-ed")
+        finally:
+            os.remove(modulemd)
+
     def buildroot_ready(self, artifacts=None):
         """
         :param artifacts=None : a list of artifacts supposed to be in the buildroot
@@ -1037,7 +1061,18 @@ class CoprModuleBuilder(GenericBuilder):
             # This forces install of bash into buildroot and srpm-buildroot
             koji add-group-pkg $module-build-tag srpm-build bash
         """
-        pass
+
+        # Start of a new batch of builds is triggered by buildsys.repo.done message.
+        # However in Copr there is no such thing. Therefore we are going to fake
+        # the message when builds are finished
+        self._send_repo_done()
+
+    def _send_repo_done(self):
+        msg = module_build_service.messaging.KojiRepoChange(
+            msg_id='a faked internal message',
+            repo_tag=self.tag_name + "-build",
+        )
+        module_build_service.scheduler.consumer.work_queue_put(msg)
 
     def buildroot_add_repos(self, dependencies):
         log.info("%r adding deps on %r" % (self, dependencies))
@@ -1099,35 +1134,15 @@ class CoprModuleBuilder(GenericBuilder):
 
         return response.data["ids"][0], koji.BUILD_STATES["BUILDING"], response.message, None
 
-    def _wait_until_all_builds_are_finished(self, module):
-        while True:
-            states = {b: self.client.get_build_details(b.task_id).status for b in module.component_builds}
-            if "failed" in states.values():
-                raise ValueError("Some builds failed")
-
-            if not filter(lambda x: x != "succeeded", states.values()):
-                return
-
-            seconds = 60
-            log.info("Going to sleep for {}s to wait until builds in copr are finished".format(seconds))
-            time.sleep(seconds)
-
     def finalize(self):
         modulemd = tempfile.mktemp()
         m1 = ModuleBuild.query.filter(ModuleBuild.name == self.module_str).one()
         m1.mmd().dump(modulemd)
 
-        # Wait until all builds are finished
-        # We shouldn't do this once the fedmsg on copr is done
-        from copr.exceptions import CoprRequestException
-        try:
-            self._wait_until_all_builds_are_finished(m1)
-        except (CoprRequestException, ValueError):
-            return log.info("Missing builds, not going to create a module")
-
         # Create a module from previous project
-        kwargs = {"username": self.copr.username, "projectname": self.copr.projectname, "modulemd": modulemd}
-        result = self.client.create_new_build_module(**kwargs)
+        result = self.client.make_module(username=self.copr.username, projectname=self.copr.projectname,
+                                         modulemd=modulemd, create=False, build=True)
+        os.remove(modulemd)
         if result.output != "ok":
             log.error(result.error)
             return

module_build_service/messaging.py

@@ -216,11 +216,12 @@ class BaseMessage(object):
                     msg_id, msg_inner_msg.get('id'), msg_inner_msg.get('state'))
 
             elif conf.system == category == 'copr' and object == 'build':
+                copr = msg_inner_msg.get('copr')
                 build = msg_inner_msg.get('build')
                 status = msg_inner_msg.get('status')
                 pkg = msg_inner_msg.get('pkg')
                 what = msg_inner_msg.get('what')
-                msg_obj = CoprBuildEnd(msg_id, build, status, pkg, what)
+                msg_obj = CoprBuildEnd(msg_id, build, status, copr, pkg, what)
 
             # If the message matched the regex and is important to the app,
             # it will be returned
@@ -271,20 +272,26 @@ class KojiRepoChange(BaseMessage):
         self.repo_tag = repo_tag
 
 
-class CoprBuildEnd(object):
-    """ A wrapper class that transforms copr message attributes
-    to a KojiBuildChange message object
+class CoprBuildEnd(KojiBuildChange):
+    """ A class that inherits from KojiBuildChange to provide a message
+     object for a build info from Copr
+
+     @TODO There should be a base class for CoprBuildEnd and KojiBuildChange
+     and conditions in the code should check for it's descendants instead of KojiBuildChange directly.
+     In such case this class would not have to inherit from koji class
+
     :param msg_id: the id of the msg (e.g. 2016-SomeGUID)
     :param build_id: the id of the build (e.g. 264382)
     :param status: the new build state
     (see http://copr-backend.readthedocs.io/package/constants.html#backend.constants.BuildStatus )
+    :param copr: the project name
     :param pkg: the full name of what is being built
     (e.g. mutt-kz-1.5.23.1-1.20150203.git.c8504a8a.fc21)
     :param state_reason: the optional reason as to why the state changed
     """
-    def __new__(cls, msg_id, build_id, status, pkg, what=None):
+    def __init__(self, msg_id, build_id, status, copr, pkg, what=None):
         nvr = kobo.rpmlib.parse_nvra(pkg)
-        return KojiBuildChange(
+        super(CoprBuildEnd, self).__init__(
             msg_id=msg_id,
             build_id=build_id,
             task_id=build_id,
@@ -294,6 +301,7 @@ class CoprBuildEnd(object):
             build_release=".".join(s for s in [nvr["release"], nvr["epoch"], nvr["arch"]] if s),
             state_reason=what,
         )
+        self.copr = copr
 
 
 class MBSModule(BaseMessage):

module_build_service/models.py

@@ -375,7 +375,7 @@ class ComponentBuild(MBSBase):
 
     @classmethod
     def from_component_event(cls, session, event):
-        if type(event) == module_build_service.messaging.KojiBuildChange:
+        if isinstance(event, module_build_service.messaging.KojiBuildChange):
             if event.module_build_id:
                 return session.query(cls).filter_by(
                     task_id=event.task_id, module_id=event.module_build_id)\

module_build_service/scheduler/consumer.py

@@ -178,7 +178,7 @@ class MBSConsumer(fedmsg.consumers.FedmsgConsumer):
         build = None
 
         # Choose a handler for this message
-        if type(msg) == module_build_service.messaging.KojiBuildChange:
+        if isinstance(msg, module_build_service.messaging.KojiBuildChange):
             handler = self.on_build_change[msg.build_new_state]
             build = models.ComponentBuild.from_component_event(session, msg)
             if build: