From bcfead0809a3a1e8ee359786b769bf119e1610fa Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Tue, 18 Dec 2018 08:06:31 +0100 Subject: [PATCH] In case module is stuck in 'init' state for more than 10 minutes, send fake MBSModule msg. This fixes issues when UMB message delivery from frontend to backend fails for whatever reason... We do the same thing for 'wait' state already, so this commit just extends it to 'init' state too. --- module_build_service/scheduler/producer.py | 27 ++++++++++++++-------- tests/test_scheduler/test_poller.py | 10 ++++---- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/module_build_service/scheduler/producer.py b/module_build_service/scheduler/producer.py index 57bf292b..62f06bb6 100644 --- a/module_build_service/scheduler/producer.py +++ b/module_build_service/scheduler/producer.py @@ -194,17 +194,22 @@ class MBSProducer(PollingProducer): log.info(' * {0} components in batch {1}' .format(n, i)) - def process_waiting_module_builds(self, session): - log.info('Looking for module builds stuck in the wait state') - builds = models.ModuleBuild.by_state(session, 'wait') - log.info(' {0!r} module builds in the wait state...' - .format(len(builds))) + def _nudge_module_builds_in_state(self, session, state, older_than_minutes): + """ + Finds all the module builds in the `state` with `time_modified` older + than `older_than_minutes` and adds fake MBSModule message to the + work queue. + """ + log.info('Looking for module builds stuck in the %s state', state) + builds = models.ModuleBuild.by_state(session, state) + log.info(' {0!r} module builds in the %s state...' + .format(len(builds), state)) now = datetime.utcnow() - ten_minutes = timedelta(minutes=10) + time_modified_threshold = timedelta(minutes=older_than_minutes) for build in builds: # Only give builds a nudge if stuck for more than ten minutes - if (now - build.time_modified) < ten_minutes: + if (now - build.time_modified) < time_modified_threshold: continue # Pretend the build is modified, so we don't tight spin. @@ -212,12 +217,16 @@ class MBSProducer(PollingProducer): session.commit() # Fake a message to kickstart the build anew in the consumer - state = module_build_service.models.BUILD_STATES['wait'] + state = module_build_service.models.BUILD_STATES[state] msg = module_build_service.messaging.MBSModule( - 'fake message', build.id, state) + 'nudge_module_builds_fake_message', build.id, state) log.info(" Scheduling faked event %r" % msg) module_build_service.scheduler.consumer.work_queue_put(msg) + def process_waiting_module_builds(self, session): + for state in ['init', 'wait']: + self._nudge_module_builds_in_state(session, state, 10) + def process_open_component_builds(self, session): log.warning('process_open_component_builds is not yet implemented...') diff --git a/tests/test_scheduler/test_poller.py b/tests/test_scheduler/test_poller.py index e1f0858c..b486c9f5 100644 --- a/tests/test_scheduler/test_poller.py +++ b/tests/test_scheduler/test_poller.py @@ -356,8 +356,9 @@ class TestPoller: koji_session.deleteBuildTarget.assert_not_called() + @pytest.mark.parametrize('state', ['init', 'wait']) def test_process_waiting_module_build( - self, create_builder, global_consumer, dbg): + self, create_builder, global_consumer, dbg, state): """ Test that processing old waiting module builds works. """ consumer = mock.MagicMock() @@ -370,7 +371,7 @@ class TestPoller: # Change the batch to 2, so the module build is in state where # it is not building anything, but the state is "build". module_build = models.ModuleBuild.query.filter_by(id=3).one() - module_build.state = 1 + module_build.state = models.BUILD_STATES[state] original = datetime.utcnow() - timedelta(minutes=11) module_build.time_modified = original db.session.commit() @@ -387,8 +388,9 @@ class TestPoller: # ensure the time_modified was changed. assert module_build.time_modified > original + @pytest.mark.parametrize('state', ['init', 'wait']) def test_process_waiting_module_build_not_old_enough( - self, create_builder, global_consumer, dbg): + self, create_builder, global_consumer, dbg, state): """ Test that we do not process young waiting builds. """ consumer = mock.MagicMock() @@ -401,7 +403,7 @@ class TestPoller: # Change the batch to 2, so the module build is in state where # it is not building anything, but the state is "build". module_build = models.ModuleBuild.query.filter_by(id=3).one() - module_build.state = 1 + module_build.state = models.BUILD_STATES[state] original = datetime.utcnow() - timedelta(minutes=9) module_build.time_modified = original db.session.commit()