check Koji active tasks for component relicts (same NVR) from previous (terminated) builds

module_build_service/builder.py

@@ -299,6 +299,19 @@ class GenericBuilder(six.with_metaclass(ABCMeta)):
             raise
         return groups
 
+    @abstractmethod
+    def list_tasks_for_components(self, component_builds=None, state='active'):
+        """
+        :param component_builds: list of component builds which we want to check
+        :param state: limit the check only for tasks in the given state
+        :return: list of tasks
+
+        This method is supposed to list tasks ('active' by default)
+        for component builds.
+        """
+        raise NotImplementedError()
+
+
 class KojiModuleBuilder(GenericBuilder):
     """ Koji specific builder class """
 
@@ -634,7 +647,10 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
             else:
                 module_target = self.module_target['name']
 
-            build_opts = {"skip_tag": True}
+            build_opts = {"skip_tag": True,
+                          "mbs_artifact_name": artifact_name,
+                          "mbs_module_name": module_target}
+
             task_id = self.koji_session.build(source, module_target, build_opts,
                                             priority=self.build_priority)
             log.info("submitted build of %s (task_id=%s), via %s" % (
@@ -830,6 +846,45 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
 
         return self.koji_session.getBuildTarget(name)
 
+    def list_tasks_for_components(self, component_builds=None, state='active'):
+        """
+        :param component_builds: list of component builds which we want to check
+        :param state: limit the check only for Koji tasks in the given state
+        :return: list of Koji tasks
+
+        List Koji tasks ('active' by default) for component builds.
+        """
+
+        component_builds = component_builds or []
+        if state == 'active':
+            states = [koji.TASK_STATES['FREE'],
+                      koji.TASK_STATES['OPEN'],
+                      koji.TASK_STATES['ASSIGNED']]
+        elif state.upper() in koji.TASK_STATES:
+            states = [koji.TASK_STATES[state.upper()]]
+        else:
+            raise ValueError("State {} is not valid within Koji task states."
+                             .format(state))
+
+        tasks = []
+        for task in self.koji_session.listTasks(opts={'state': states,
+                                                      'decode': True,
+                                                      'method': 'build'}):
+            task_opts = task['request'][-1]
+            assert isinstance(task_opts, dict), "Task options shall be a dict."
+            if 'scratch' in task_opts and task_opts['scratch']:
+                continue
+            if 'mbs_artifact_name' not in task_opts:
+                task_opts['mbs_artifact_name'] = None
+            if 'mbs_module_name' not in task_opts:
+                task_opts['mbs_module_name'] = None
+            for c in component_builds:
+                if (c.name == task_opts['mbs_artifact_name'] and
+                   c.tag == task_opts['mbs_module_name']):
+                    tasks.append(task)
+
+        return tasks
+
 
 class CoprModuleBuilder(GenericBuilder):
 
@@ -1414,6 +1469,10 @@ mdpolicy=group:primary
     def cancel_build(self, task_id):
         pass
 
+    def list_tasks_for_components(self, component_builds=None, state='active'):
+        pass
+
+
 def build_from_scm(artifact_name, source, config, build_srpm,
                         data = None, stdout=None, stderr=None):
     """

module_build_service/utils.py

@@ -92,6 +92,7 @@ def start_build_batch(config, module, session, builder, components=None):
     """
     import koji  # Placed here to avoid py2/py3 conflicts...
 
+    # Local check for component relicts
     if any([c.state == koji.BUILD_STATES['BUILDING']
             for c in module.component_builds]):
         err_msg = "Cannot start a batch when another is in flight."
@@ -103,6 +104,22 @@ def start_build_batch(config, module, session, builder, components=None):
         log.error("Components in building state: %s" % str(unbuilt_components))
         raise ValueError(err_msg)
 
+    # Identify active tasks which might contain relicts of previous builds
+    # and fail the module build if this^ happens.
+    active_tasks = builder.list_tasks_for_components(module.component_builds,
+                                                     state='active')
+    if isinstance(active_tasks, list) and active_tasks:
+        state_reason = "Cannot start a batch, because some components are already in 'building' state."
+        state_reason += " See tasks (ID): {}".format(', '.join([str(t['id']) for t in active_tasks]))
+        module.transition(config, state=models.BUILD_STATES['failed'],
+                          state_reason=state_reason)
+        session.commit()
+        return
+
+    else:
+        log.debug("Builder {} doesn't provide information about active tasks."
+                  .format(builder))
+
     # The user can either pass in a list of components to 'seed' the batch, or
     # if none are provided then we just select everything that hasn't
     # successfully built yet or isn't currently being built.

tests/test_build/test_build.py

@@ -192,6 +192,9 @@ class TestModuleBuilder(GenericBuilder):
         if TestModuleBuilder.on_cancel_cb:
             TestModuleBuilder.on_cancel_cb(self, task_id)
 
+    def list_tasks_for_components(self, component_builds=None, state='active'):
+        pass
+
 
 class TestBuild(unittest.TestCase):
 

tests/test_scheduler/test_repo_done.py

@@ -57,11 +57,12 @@ class TestRepoDone(unittest.TestCase):
         module_build_service.scheduler.handlers.repos.done(
             config=conf, session=db.session, msg=msg)
 
+    @mock.patch('module_build_service.builder.KojiModuleBuilder.list_tasks_for_components', return_value=[])
     @mock.patch('module_build_service.builder.KojiModuleBuilder.buildroot_ready', return_value=True)
     @mock.patch('module_build_service.builder.KojiModuleBuilder.get_session')
     @mock.patch('module_build_service.builder.KojiModuleBuilder.build')
     @mock.patch('module_build_service.builder.KojiModuleBuilder.buildroot_connect')
-    def test_a_single_match(self, connect, build_fn, get_session, ready):
+    def test_a_single_match(self, connect, build_fn, get_session, ready, list_tasks_fn):
         """ Test that when a repo msg hits us and we have a single match.
         """
         get_session.return_value = mock.Mock(), 'development'
@@ -75,11 +76,12 @@ class TestRepoDone(unittest.TestCase):
             artifact_name='communicator',
             source='git://pkgs.domain.local/rpms/communicator?#da95886c8a443b36a9ce31abda1f9bed22f2f9c2')
 
+    @mock.patch('module_build_service.builder.KojiModuleBuilder.list_tasks_for_components', return_value=[])
     @mock.patch('module_build_service.builder.KojiModuleBuilder.buildroot_ready', return_value=True)
     @mock.patch('module_build_service.builder.KojiModuleBuilder.get_session')
     @mock.patch('module_build_service.builder.KojiModuleBuilder.build')
     @mock.patch('module_build_service.builder.KojiModuleBuilder.buildroot_connect')
-    def test_a_single_match_build_fail(self, connect, build_fn, config, ready):
+    def test_a_single_match_build_fail(self, connect, build_fn, config, ready, list_tasks_fn):
         """ Test that when a KojiModuleBuilder.build fails, the build is
         marked as failed with proper state_reason.
         """