From d06c13d9739c6564b41e277484129626dc08de5a Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Mon, 12 Dec 2016 21:02:57 +0100 Subject: [PATCH] Fix the bug when module build freezes when all the components in batch have been already built by builder. --- module_build_service/manage.py | 4 ++ module_build_service/messaging.py | 29 ++++++++++++ module_build_service/models.py | 4 +- .../scheduler/handlers/repos.py | 8 +++- module_build_service/scheduler/main.py | 7 ++- module_build_service/utils.py | 22 +++++++++- tests/test_build/test_build.py | 44 ++++++++++++++++++- 7 files changed, 110 insertions(+), 8 deletions(-) diff --git a/module_build_service/manage.py b/module_build_service/manage.py index 0e2bc535..e3a6f178 100644 --- a/module_build_service/manage.py +++ b/module_build_service/manage.py @@ -40,6 +40,7 @@ from module_build_service.utils import ( insert_fake_baseruntime, ) from module_build_service.messaging import RidaModule +import module_build_service.messaging manager = Manager(app) @@ -292,6 +293,9 @@ def runssl(host=conf.host, port=conf.port, debug=conf.debug): """ Runs the Flask app with the HTTPS settings configured in config.py """ logging.info('Starting Module Build Service frontend') + + module_build_service.messaging.init(conf) + ssl_ctx = _establish_ssl_context() app.run( host=host, diff --git a/module_build_service/messaging.py b/module_build_service/messaging.py index 573f7317..33da8c57 100644 --- a/module_build_service/messaging.py +++ b/module_build_service/messaging.py @@ -228,6 +228,17 @@ class RidaModule(BaseMessage): self.module_build_id = module_build_id self.module_build_state = module_build_state +def init(conf, **kwargs): + """ + Initialize the messaging backend. + :param conf: a Config object from the class in config.py + :param kwargs: any additional arguments to pass to the backend handler + """ + try: + handler = _messaging_backends[conf.messaging]['init'] + except KeyError: + raise KeyError("No messaging backend found for %r" % conf.messaging) + return handler(conf, **kwargs) def publish(topic, msg, conf, service): """ @@ -336,6 +347,15 @@ _in_memory_work_queue = queue.Queue() # Message id for "in_memory" messaging. _in_memory_msg_id = 0 +def _in_memory_init(conf, **kwargs): + """ + Initializes the In Memory messaging backend. + """ + global _in_memory_work_queue + global _in_memory_msg_id + _in_memory_msg_id = 0 + _in_memory_work_queue = queue.Queue() + def _in_memory_publish(topic, msg, conf, service): """ Puts the message to _in_memory_work_queue". @@ -364,16 +384,25 @@ def _in_memory_listen(conf, **kwargs): while True: yield _in_memory_work_queue.get(True) +def _no_op(conf, **kwargs): + """ + No operation. + """ + pass + _messaging_backends = { 'fedmsg': { + 'init': _no_op, 'publish': _fedmsg_publish, 'listen': _fedmsg_listen, }, 'amq': { + 'init': _no_op, 'publish': _amq_publish, 'listen': _amq_listen, }, 'in_memory': { + 'init': _in_memory_init, 'publish': _in_memory_publish, 'listen': _in_memory_listen, }, diff --git a/module_build_service/models.py b/module_build_service/models.py index d0b2558d..38996cc7 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -329,9 +329,9 @@ class ModuleBuild(RidaBase): return result def __repr__(self): - return "" % ( + return "" % ( self.name, self.stream, self.version, - INVERSE_BUILD_STATES[self.state], self.batch) + INVERSE_BUILD_STATES[self.state], self.batch, self.state_reason) class ComponentBuild(RidaBase): diff --git a/module_build_service/scheduler/handlers/repos.py b/module_build_service/scheduler/handlers/repos.py index ce776b1a..5010d086 100644 --- a/module_build_service/scheduler/handlers/repos.py +++ b/module_build_service/scheduler/handlers/repos.py @@ -118,6 +118,7 @@ def done(config, session, msg): and c.state != koji.BUILD_STATES["FAILED"]) ] + further_work = [] if unbuilt_components: # Increment the build batch when no components are being built and all # have at least attempted a build (even failures) in the current batch @@ -128,15 +129,18 @@ def done(config, session, msg): if not unbuilt_components_in_batch: module_build.batch += 1 - module_build_service.utils.start_build_batch( + further_work += module_build_service.utils.start_build_batch( config, module_build, session, builder) # We don't have copr implementation finished yet, Let's fake the repo change event, # as if copr builds finished successfully if config.system == "copr": - return [module_build_service.messaging.KojiRepoChange('fake msg', module_build.koji_tag)] + further_work += [module_build_service.messaging.KojiRepoChange('fake msg', module_build.koji_tag)] + return further_work else: module_build.transition(config, state=models.BUILD_STATES['done']) session.commit() builder.finalize() + + return further_work diff --git a/module_build_service/scheduler/main.py b/module_build_service/scheduler/main.py index 274726e9..faf1b62d 100644 --- a/module_build_service/scheduler/main.py +++ b/module_build_service/scheduler/main.py @@ -300,8 +300,11 @@ class Poller(threading.Thread): # If there are no components in the build state on the module build, # then no possible event will start off new component builds if not module_build.current_batch(koji.BUILD_STATES['BUILDING']): - module_build_service.utils.start_build_batch( + further_work = module_build_service.utils.start_build_batch( config, module_build, session, config.system) + for event in further_work: + log.info(" Scheduling faked event %r" % event) + self.outgoing_work_queue.put(event) _work_queue = queue.Queue() @@ -326,6 +329,8 @@ def graceful_stop(): def main(initial_msgs=[], return_after_build=False): log.info("Starting module_build_service_daemon.") + module_build_service.messaging.init(conf) + for msg in initial_msgs: outgoing_work_queue_put(msg) diff --git a/module_build_service/utils.py b/module_build_service/utils.py index e18074e7..3c0785b8 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -39,6 +39,7 @@ from module_build_service import log, models from module_build_service.errors import ValidationError, UnprocessableEntity from module_build_service import conf, db from module_build_service.errors import (Unauthorized, Conflict) +import module_build_service.messaging from multiprocessing.dummy import Pool as ThreadPool @@ -84,7 +85,12 @@ def at_concurrent_component_threshold(config, session): def start_build_batch(config, module, session, builder, components=None): - """ Starts a round of the build cycle for a module. """ + """ + Starts a round of the build cycle for a module. + + Returns list of BaseMessage instances which should be scheduled by the + scheduler. + """ import koji # Placed here to avoid py2/py3 conflicts... if any([c.state == koji.BUILD_STATES['BUILDING'] @@ -123,8 +129,20 @@ def start_build_batch(config, module, session, builder, components=None): c.state_reason = "Failed to submit artifact %s to Koji" % (c.package) continue - session.commit() + further_work = [] + # If all components in this batch are already done, it can mean that they + # have been built in the past and have been skipped in this module build. + # We therefore have to generate fake KojiRepoChange message, because the + # repo has been also done in the past and build system will not send us + # any message now. + if (all(c.state == koji.BUILD_STATES['COMPLETE'] for c in unbuilt_components) + and builder.module_build_tag): + further_work += [module_build_service.messaging.KojiRepoChange( + 'start_build_batch: fake msg', builder.module_build_tag['name'])] + + session.commit() + return further_work def pagination_metadata(p_query): """ diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 5af0c84e..f3b0b22a 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -80,6 +80,7 @@ class TestModuleBuilder(GenericBuilder): _build_id = 1 BUILD_STATE = "COMPLETE" + INSTANT_COMPLETE = False on_build_cb = None on_cancel_cb = None @@ -92,6 +93,7 @@ class TestModuleBuilder(GenericBuilder): @classmethod def reset(cls): TestModuleBuilder.BUILD_STATE = "COMPLETE" + TestModuleBuilder.INSTANT_COMPLETE = False TestModuleBuilder.on_build_cb = None TestModuleBuilder.on_cancel_cb = None @@ -116,6 +118,10 @@ class TestModuleBuilder(GenericBuilder): def buildroot_add_repos(self, dependencies): pass + @property + def module_build_tag(self): + return {"name": self.tag_name + "-build"} + def _send_repo_done(self): msg = module_build_service.messaging.KojiRepoChange( msg_id='a faked internal message', @@ -152,7 +158,11 @@ class TestModuleBuilder(GenericBuilder): if TestModuleBuilder.on_build_cb: TestModuleBuilder.on_build_cb(self, artifact_name, source) - state = koji.BUILD_STATES['BUILDING'] + if TestModuleBuilder.INSTANT_COMPLETE: + state = koji.BUILD_STATES['COMPLETE'] + else: + state = koji.BUILD_STATES['BUILDING'] + reason = "Submitted %s to Koji" % (artifact_name) return TestModuleBuilder._build_id, state, reason, None @@ -262,3 +272,35 @@ class TestBuild(unittest.TestCase): # Check that cancel_build has been called for this build if build.task_id: self.assertTrue(build.task_id in cancelled_tasks) + + @timed(30) + @patch('module_build_service.auth.get_username', return_value='Homer J. Simpson') + @patch('module_build_service.auth.assert_is_packager') + @patch('module_build_service.scm.SCM') + def test_submit_build_instant_complete(self, mocked_scm, mocked_assert_is_packager, + mocked_get_username): + """ + Tests the build of testmodule.yaml using TestModuleBuilder which + succeeds everytime. + """ + mocked_scm_obj = MockedSCM(mocked_scm, "testmodule", "testmodule.yaml") + + rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( + {'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' + 'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'})) + + data = json.loads(rv.data) + module_build_id = data['id'] + + TestModuleBuilder.BUILD_STATE = "BUILDING" + TestModuleBuilder.INSTANT_COMPLETE = True + + msgs = [] + msgs.append(RidaModule("fake msg", 1, 1)) + module_build_service.scheduler.main.main(msgs, True) + + # All components should be built and module itself should be in "done" + # or "ready" state. + for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all(): + self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE']) + self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] )