From 22758419c0a360442384beb115993ff71b62a265 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Wed, 10 May 2017 22:20:10 +0200 Subject: [PATCH] Delete build target after config.koji_target_delete_time seconds. --- .../builder/KojiModuleBuilder.py | 5 -- module_build_service/config.py | 5 ++ module_build_service/scheduler/producer.py | 42 +++++++++++- tests/test_scheduler/test_poller.py | 64 +++++++++++++++++++ 4 files changed, 110 insertions(+), 6 deletions(-) diff --git a/module_build_service/builder/KojiModuleBuilder.py b/module_build_service/builder/KojiModuleBuilder.py index 8350995e..a176eda8 100644 --- a/module_build_service/builder/KojiModuleBuilder.py +++ b/module_build_service/builder/KojiModuleBuilder.py @@ -262,11 +262,6 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules self.module_build_tag, self.module_tag) - # Add -repo target, so Kojira creates RPM repository with built - # module for us. - self._koji_add_target(self.tag_name + "-repo", self.module_tag, - self.module_tag) - self.__prep = True log.info("%r buildroot sucessfully connected." % self) diff --git a/module_build_service/config.py b/module_build_service/config.py index e2e2fbe7..ca9813b5 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -181,6 +181,11 @@ class Config(object): 'type': list, 'default': ['module'], 'desc': 'List of allowed koji tag prefixes.'}, + 'koji_target_delete_time': { + 'type': int, + 'default': 24*3600, + 'desc': 'Time in seconds after which the Koji target of ' + 'built module is deleted'}, 'allow_custom_scmurls': { 'type': bool, 'default': False, diff --git a/module_build_service/scheduler/producer.py b/module_build_service/scheduler/producer.py index ccccb623..e78881bf 100644 --- a/module_build_service/scheduler/producer.py +++ b/module_build_service/scheduler/producer.py @@ -26,7 +26,7 @@ fedmsg-hub. This class polls the database for tasks to do. import koji import operator -from datetime import timedelta +from datetime import timedelta, datetime from sqlalchemy.orm import lazyload from moksha.hub.api.producer import PollingProducer @@ -50,6 +50,7 @@ class MBSProducer(PollingProducer): self.fail_lost_builds(session) self.process_paused_module_builds(conf, session) self.trigger_new_repo_when_stalled(conf, session) + self.delete_old_koji_targets(conf, session) except Exception as e: msg = 'Error in poller execution:' log.exception(msg) @@ -239,3 +240,42 @@ class MBSProducer(PollingProducer): module_build.new_repo_task_id = 0 session.commit() + + def delete_old_koji_targets(self, config, session): + """ + Deletes targets older than `config.koji_target_delete_time` seconds + from Koji to cleanup after the module builds. + """ + if config.system != 'koji': + return + + log.info('Looking for module builds which Koji target can be removed') + + now = datetime.utcnow() + + koji_session = module_build_service.builder.KojiModuleBuilder\ + .get_session(config, None) + for target in koji_session.getBuildTargets(): + koji_tag = target["dest_tag_name"] + module = session.query(models.ModuleBuild).filter_by( + koji_tag=koji_tag).first() + if not module or module.state in [models.BUILD_STATES["init"], + models.BUILD_STATES["wait"], + models.BUILD_STATES["build"]]: + continue + + # Double-check that the target we are going to remove is prefixed + # by our prefix, so we won't remove f26 when there is some garbage + # in DB or Koji. + for allowed_prefix in config.koji_tag_prefixes: + if target['name'].startswith(allowed_prefix + "-"): + break + else: + log.error("Module %r has Koji target with not allowed prefix.", + module) + continue + + delta = now - module.time_completed + if delta.total_seconds() > config.koji_target_delete_time: + log.info("Removing target of module %r", module) + koji_session.deleteBuildTarget(target['id']) diff --git a/tests/test_scheduler/test_poller.py b/tests/test_scheduler/test_poller.py index 26400b7d..3e5eb025 100644 --- a/tests/test_scheduler/test_poller.py +++ b/tests/test_scheduler/test_poller.py @@ -35,6 +35,7 @@ import module_build_service.scheduler.handlers.components from module_build_service.builder import GenericBuilder, KojiModuleBuilder from module_build_service.scheduler.producer import MBSProducer import six.moves.queue as queue +from datetime import datetime, timedelta BASE_DIR = path.abspath(path.dirname(__file__)) CASSETTES_DIR = path.join( @@ -195,3 +196,66 @@ class TestPoller(unittest.TestCase): components = module_build.current_batch() for component in components: self.assertEqual(component.state, None) + + def test_delete_old_koji_targets( + self, create_builder, koji_get_session, global_consumer, dbg): + """ + Tests that we delete koji target when time_completed is older than + koji_target_delete_time value. + """ + consumer = mock.MagicMock() + consumer.incoming = queue.Queue() + global_consumer.return_value = consumer + + for state_name, state in models.BUILD_STATES.items(): + koji_session = mock.MagicMock() + koji_session.getBuildTargets.return_value = [ + {'dest_tag_name': 'module-tag', 'id': 852, 'name': 'module-tag'}, + {'dest_tag_name': 'f26', 'id': 853, 'name': 'f26'}, + {'dest_tag_name': 'module-tag2', 'id': 853, 'name': 'f26'}] + koji_get_session.return_value = koji_session + + builder = mock.MagicMock() + create_builder.return_value = builder + + # Change the batch to 2, so the module build is in state where + # it is not building anything, but the state is "build". + module_build = models.ModuleBuild.query.filter_by(id=2).one() + module_build.state = state + module_build.koji_tag = "module-tag" + module_build.time_completed = datetime.utcnow() + module_build.new_repo_task_id = 123456 + db.session.commit() + + # Poll :) + hub = mock.MagicMock() + poller = MBSProducer(hub) + poller.delete_old_koji_targets(conf, db.session) + + db.session.refresh(module_build) + module_build.time_completed = datetime.utcnow() - timedelta(hours=23) + db.session.commit() + poller.delete_old_koji_targets(conf, db.session) + + # deleteBuildTarget should not be called, because time_completed is + # set to "now". + self.assertTrue(not koji_session.deleteBuildTarget.called) + + # Try removing non-modular target - should not happen + db.session.refresh(module_build) + module_build.koji_tag = "module-tag2" + module_build.time_completed = datetime.utcnow() - timedelta(hours=25) + db.session.commit() + poller.delete_old_koji_targets(conf, db.session) + self.assertTrue(not koji_session.deleteBuildTarget.called) + + # Refresh our module_build object and set time_completed 25 hours ago + db.session.refresh(module_build) + module_build.time_completed = datetime.utcnow() - timedelta(hours=25) + module_build.koji_tag = "module-tag" + db.session.commit() + + poller.delete_old_koji_targets(conf, db.session) + + if state_name in ["done", "ready", "failed"]: + koji_session.deleteBuildTarget.assert_called_once_with(852)