From cdf06ad999c03696b83dfeb26922716bb7ee8478 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Fri, 7 Apr 2017 10:13:11 +0200 Subject: [PATCH] Start newRepo only when all the COMPLETE components have been tagged and there are no unbuilt components in a batch --- .../scheduler/handlers/tags.py | 17 +++- tests/test_messaging.py | 20 +++++ tests/test_scheduler/test_tag_tagged.py | 86 +++++++++++++++++++ 3 files changed, 121 insertions(+), 2 deletions(-) diff --git a/module_build_service/scheduler/handlers/tags.py b/module_build_service/scheduler/handlers/tags.py index b2b381be..e80aafe7 100644 --- a/module_build_service/scheduler/handlers/tags.py +++ b/module_build_service/scheduler/handlers/tags.py @@ -56,14 +56,27 @@ def tagged(config, session, msg): log.error("No component %s in module %r", msg.artifact, module_build) return + log.info("Saw relevant component tag of %r from %r." % (component.nvr, + msg.msg_id)) + # Mark the component as tagged component.tagged = True session.commit() - # Get the list of untagged components in current batch. + unbuilt_components_in_batch = [ + c for c in module_build.current_batch() + if c.state == koji.BUILD_STATES['BUILDING'] or not c.state + ] + if unbuilt_components_in_batch: + log.info("Not regenerating repo for tag %s, there are still " + "building components in a batch", tag) + return [] + + # Get the list of untagged components in current batch which + # have been built successfully. untagged_components = [ c for c in module_build.current_batch() - if not c.tagged + if not c.tagged and c.state == koji.BUILD_STATES['COMPLETE'] ] # If all components are tagged, start newRepo task. diff --git a/tests/test_messaging.py b/tests/test_messaging.py index 3e408a68..d10406ae 100644 --- a/tests/test_messaging.py +++ b/tests/test_messaging.py @@ -115,3 +115,23 @@ class TestFedmsgMessaging(unittest.TestCase): self.assertEqual(msg.tag, "module-debugging-tools-master-20170405115403-build") self.assertEqual(msg.artifact, "module-build-macros") + + + def test_buildsys_repo_done(self): + # https://fedora-fedmsg.readthedocs.io/en/latest/topics.html#id134 + buildsys_tag_msg = { + "msg": { + "instance": "primary", + "repo_id": 728809, + "tag": "module-f0f7e44f3c6cccab-build", + "tag_id": 653 + }, + 'msg_id': '2015-51be4c8e-8ab6-4dcb-ac0d-37b257765c71', + 'timestamp': 1424789698.0, + 'topic': 'org.fedoraproject.prod.buildsys.repo.done' + } + + topic = 'org.fedoraproject.prod.buildsys.repo.done' + msg = messaging.BaseMessage.from_fedmsg(topic, buildsys_tag_msg) + + self.assertEqual(msg.repo_tag, "module-f0f7e44f3c6cccab-build") diff --git a/tests/test_scheduler/test_tag_tagged.py b/tests/test_scheduler/test_tag_tagged.py index eb810cc5..3e87d304 100644 --- a/tests/test_scheduler/test_tag_tagged.py +++ b/tests/test_scheduler/test_tag_tagged.py @@ -85,6 +85,8 @@ class TestTagTagged(unittest.TestCase): module_build = module_build_service.models.ModuleBuild.query.filter_by(id=2).one() module_build.batch = 2 + for c in module_build.current_batch(): + c.state = koji.BUILD_STATES["COMPLETE"] db.session.commit() # Tag the first component to the buildroot. @@ -113,3 +115,87 @@ class TestTagTagged(unittest.TestCase): # newRepo task_id should be stored in database, so we can check its # status later in poller. self.assertEqual(module_build.new_repo_task_id, 123456) + + + @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", + return_value = {'build': [], 'srpm-build': []}) + @patch("module_build_service.builder.KojiModuleBuilder.get_session") + @patch("module_build_service.builder.GenericBuilder.create_from_module") + def test_newrepo_still_building_components(self, create_builder, koji_get_session, dbg): + """ + Test that newRepo is called in the expected times. + """ + koji_session = mock.MagicMock() + koji_session.getTag = lambda tag_name: {'name': tag_name} + koji_session.getTaskInfo.return_value = {'state': koji.TASK_STATES['CLOSED']} + koji_session.newRepo.return_value = 123456 + koji_get_session.return_value = koji_session + + builder = mock.MagicMock() + builder.koji_session = koji_session + builder.buildroot_ready.return_value = False + create_builder.return_value = builder + + module_build = module_build_service.models.ModuleBuild.query.filter_by(id=2).one() + module_build.batch = 2 + component = module_build_service.models.ComponentBuild.query\ + .filter_by(package='perl-Tangerine', module_id=module_build.id).one() + component.state = koji.BUILD_STATES["BUILDING"] + db.session.commit() + + # Tag the perl-List-Compare component to the buildroot. + msg = module_build_service.messaging.KojiTagChange( + 'id', 'module-testmodule-build', "perl-Tangerine") + module_build_service.scheduler.handlers.tags.tagged( + config=conf, session=db.session, msg=msg) + + # newRepo should not be called, because perl-List-Compare has not been + # built yet. + self.assertTrue(not koji_session.newRepo.called) + + @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", + return_value = {'build': [], 'srpm-build': []}) + @patch("module_build_service.builder.KojiModuleBuilder.get_session") + @patch("module_build_service.builder.GenericBuilder.create_from_module") + def test_newrepo_failed_components(self, create_builder, koji_get_session, dbg): + """ + Test that newRepo is called in the expected times. + """ + koji_session = mock.MagicMock() + koji_session.getTag = lambda tag_name: {'name': tag_name} + koji_session.getTaskInfo.return_value = {'state': koji.TASK_STATES['CLOSED']} + koji_session.newRepo.return_value = 123456 + koji_get_session.return_value = koji_session + + builder = mock.MagicMock() + builder.koji_session = koji_session + builder.buildroot_ready.return_value = False + create_builder.return_value = builder + + module_build = module_build_service.models.ModuleBuild.query.filter_by(id=2).one() + module_build.batch = 2 + component = module_build_service.models.ComponentBuild.query\ + .filter_by(package='perl-Tangerine', module_id=module_build.id).one() + component.state = koji.BUILD_STATES["FAILED"] + component = module_build_service.models.ComponentBuild.query\ + .filter_by(package='perl-List-Compare', module_id=module_build.id).one() + component.state = koji.BUILD_STATES["COMPLETE"] + db.session.commit() + + # Tag the perl-List-Compare component to the buildroot. + msg = module_build_service.messaging.KojiTagChange( + 'id', 'module-testmodule-build', "perl-List-Compare") + module_build_service.scheduler.handlers.tags.tagged( + config=conf, session=db.session, msg=msg) + + # newRepo should be called now - all successfully built + # components have been tagged. + koji_session.newRepo.assert_called_once_with("module-testmodule-build") + + # Refresh our module_build object. + db.session.expunge(module_build) + module_build = module_build_service.models.ModuleBuild.query.filter_by(id=2).one() + + # newRepo task_id should be stored in database, so we can check its + # status later in poller. + self.assertEqual(module_build.new_repo_task_id, 123456)