Delete build target after config.koji_target_delete_time seconds.

module_build_service/builder/KojiModuleBuilder.py

@@ -262,11 +262,6 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules
                                                    self.module_build_tag,
                                                    self.module_tag)
 
-        # Add -repo target, so Kojira creates RPM repository with built
-        # module for us.
-        self._koji_add_target(self.tag_name + "-repo", self.module_tag,
-                              self.module_tag)
-
         self.__prep = True
         log.info("%r buildroot sucessfully connected." % self)
 

module_build_service/config.py

@@ -181,6 +181,11 @@ class Config(object):
             'type': list,
             'default': ['module'],
             'desc': 'List of allowed koji tag prefixes.'},
+        'koji_target_delete_time': {
+            'type': int,
+            'default': 24*3600,
+            'desc': 'Time in seconds after which the Koji target of '
+                'built module is deleted'},
         'allow_custom_scmurls': {
             'type': bool,
             'default': False,

module_build_service/scheduler/producer.py

@@ -26,7 +26,7 @@ fedmsg-hub. This class polls the database for tasks to do.
 
 import koji
 import operator
-from datetime import timedelta
+from datetime import timedelta, datetime
 from sqlalchemy.orm import lazyload
 from moksha.hub.api.producer import PollingProducer
 
@@ -50,6 +50,7 @@ class MBSProducer(PollingProducer):
                 self.fail_lost_builds(session)
                 self.process_paused_module_builds(conf, session)
                 self.trigger_new_repo_when_stalled(conf, session)
+                self.delete_old_koji_targets(conf, session)
             except Exception as e:
                 msg = 'Error in poller execution:'
                 log.exception(msg)
@@ -239,3 +240,42 @@ class MBSProducer(PollingProducer):
                 module_build.new_repo_task_id = 0
 
         session.commit()
+
+    def delete_old_koji_targets(self, config, session):
+        """
+        Deletes targets older than `config.koji_target_delete_time` seconds
+        from Koji to cleanup after the module builds.
+        """
+        if config.system != 'koji':
+            return
+
+        log.info('Looking for module builds which Koji target can be removed')
+
+        now = datetime.utcnow()
+
+        koji_session = module_build_service.builder.KojiModuleBuilder\
+            .get_session(config, None)
+        for target in koji_session.getBuildTargets():
+            koji_tag = target["dest_tag_name"]
+            module = session.query(models.ModuleBuild).filter_by(
+                koji_tag=koji_tag).first()
+            if not module or module.state in [models.BUILD_STATES["init"],
+                                              models.BUILD_STATES["wait"],
+                                              models.BUILD_STATES["build"]]:
+                continue
+
+            # Double-check that the target we are going to remove is prefixed
+            # by our prefix, so we won't remove f26 when there is some garbage
+            # in DB or Koji.
+            for allowed_prefix in config.koji_tag_prefixes:
+                if target['name'].startswith(allowed_prefix + "-"):
+                    break
+            else:
+                log.error("Module %r has Koji target with not allowed prefix.",
+                          module)
+                continue
+
+            delta = now - module.time_completed
+            if delta.total_seconds() > config.koji_target_delete_time:
+                log.info("Removing target of module %r", module)
+                koji_session.deleteBuildTarget(target['id'])

tests/test_scheduler/test_poller.py

@@ -35,6 +35,7 @@ import module_build_service.scheduler.handlers.components
 from module_build_service.builder import GenericBuilder, KojiModuleBuilder
 from module_build_service.scheduler.producer import MBSProducer
 import six.moves.queue as queue
+from datetime import datetime, timedelta
 
 BASE_DIR = path.abspath(path.dirname(__file__))
 CASSETTES_DIR = path.join(
@@ -195,3 +196,66 @@ class TestPoller(unittest.TestCase):
         components = module_build.current_batch()
         for component in components:
             self.assertEqual(component.state, None)
+
+    def test_delete_old_koji_targets(
+            self, create_builder, koji_get_session, global_consumer, dbg):
+        """
+        Tests that we delete koji target when time_completed is older than
+        koji_target_delete_time value.
+        """
+        consumer = mock.MagicMock()
+        consumer.incoming = queue.Queue()
+        global_consumer.return_value = consumer
+
+        for state_name, state in models.BUILD_STATES.items():
+            koji_session = mock.MagicMock()
+            koji_session.getBuildTargets.return_value = [
+                {'dest_tag_name': 'module-tag', 'id': 852, 'name': 'module-tag'},
+                {'dest_tag_name': 'f26', 'id': 853, 'name': 'f26'},
+                {'dest_tag_name': 'module-tag2', 'id': 853, 'name': 'f26'}]
+            koji_get_session.return_value = koji_session
+
+            builder = mock.MagicMock()
+            create_builder.return_value = builder
+
+            # Change the batch to 2, so the module build is in state where
+            # it is not building anything, but the state is "build".
+            module_build = models.ModuleBuild.query.filter_by(id=2).one()
+            module_build.state = state
+            module_build.koji_tag = "module-tag"
+            module_build.time_completed = datetime.utcnow()
+            module_build.new_repo_task_id = 123456
+            db.session.commit()
+
+            # Poll :)
+            hub = mock.MagicMock()
+            poller = MBSProducer(hub)
+            poller.delete_old_koji_targets(conf, db.session)
+
+            db.session.refresh(module_build)
+            module_build.time_completed = datetime.utcnow() - timedelta(hours=23)
+            db.session.commit()
+            poller.delete_old_koji_targets(conf, db.session)
+
+            # deleteBuildTarget should not be called, because time_completed is
+            # set to "now".
+            self.assertTrue(not koji_session.deleteBuildTarget.called)
+
+            # Try removing non-modular target - should not happen
+            db.session.refresh(module_build)
+            module_build.koji_tag = "module-tag2"
+            module_build.time_completed = datetime.utcnow() - timedelta(hours=25)
+            db.session.commit()
+            poller.delete_old_koji_targets(conf, db.session)
+            self.assertTrue(not koji_session.deleteBuildTarget.called)
+
+            # Refresh our module_build object and set time_completed 25 hours ago
+            db.session.refresh(module_build)
+            module_build.time_completed = datetime.utcnow() - timedelta(hours=25)
+            module_build.koji_tag = "module-tag"
+            db.session.commit()
+
+            poller.delete_old_koji_targets(conf, db.session)
+
+            if state_name in ["done", "ready", "failed"]:
+                koji_session.deleteBuildTarget.assert_called_once_with(852)