modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-02-11 09:05:00 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

In case module is stuck in 'init' state for more than 10 minutes, send fake MBSModule msg.

Browse Source 
This fixes issues when UMB message delivery from frontend to backend fails
for whatever reason...

We do the same thing for 'wait' state already, so this commit just extends
it to 'init' state too.
This commit is contained in:
Jan Kaluza
2018-12-18 08:06:31 +01:00
parent cc303854ad
commit bcfead0809
2 changed files with 24 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
module_build_service/scheduler/producer.py
View File

@@ -194,17 +194,22 @@ class MBSProducer(PollingProducer):
log.info(' * {0} components in batch {1}'
.format(n, i))
def process_waiting_module_builds(self, session):
log.info('Looking for module builds stuck in the wait state')
builds = models.ModuleBuild.by_state(session, 'wait')
log.info(' {0!r} module builds in the wait state...'
.format(len(builds)))
def _nudge_module_builds_in_state(self, session, state, older_than_minutes):
"""
Finds all the module builds in the `state` with `time_modified` older
than `older_than_minutes` and adds fake MBSModule message to the
work queue.
"""
log.info('Looking for module builds stuck in the %s state', state)
builds = models.ModuleBuild.by_state(session, state)
log.info(' {0!r} module builds in the %s state...'
.format(len(builds), state))
now = datetime.utcnow()
ten_minutes = timedelta(minutes=10)
time_modified_threshold = timedelta(minutes=older_than_minutes)
for build in builds:
# Only give builds a nudge if stuck for more than ten minutes
if (now - build.time_modified) < ten_minutes:
if (now - build.time_modified) < time_modified_threshold:
continue
# Pretend the build is modified, so we don't tight spin.
@@ -212,12 +217,16 @@ class MBSProducer(PollingProducer):
session.commit()
# Fake a message to kickstart the build anew in the consumer
state = module_build_service.models.BUILD_STATES['wait']
state = module_build_service.models.BUILD_STATES[state]
msg = module_build_service.messaging.MBSModule(
'fake message', build.id, state)
'nudge_module_builds_fake_message', build.id, state)
log.info(" Scheduling faked event %r" % msg)
module_build_service.scheduler.consumer.work_queue_put(msg)
def process_waiting_module_builds(self, session):
for state in ['init', 'wait']:
self._nudge_module_builds_in_state(session, state, 10)
def process_open_component_builds(self, session):
log.warning('process_open_component_builds is not yet implemented...')

10
tests/test_scheduler/test_poller.py
View File

@@ -356,8 +356,9 @@ class TestPoller:
koji_session.deleteBuildTarget.assert_not_called()
@pytest.mark.parametrize('state', ['init', 'wait'])
def test_process_waiting_module_build(
self, create_builder, global_consumer, dbg):
self, create_builder, global_consumer, dbg, state):
""" Test that processing old waiting module builds works. """
consumer = mock.MagicMock()
@@ -370,7 +371,7 @@ class TestPoller:
# Change the batch to 2, so the module build is in state where
# it is not building anything, but the state is "build".
module_build = models.ModuleBuild.query.filter_by(id=3).one()
module_build.state = 1
module_build.state = models.BUILD_STATES[state]
original = datetime.utcnow() - timedelta(minutes=11)
module_build.time_modified = original
db.session.commit()
@@ -387,8 +388,9 @@ class TestPoller:
# ensure the time_modified was changed.
assert module_build.time_modified > original
@pytest.mark.parametrize('state', ['init', 'wait'])
def test_process_waiting_module_build_not_old_enough(
self, create_builder, global_consumer, dbg):
self, create_builder, global_consumer, dbg, state):
""" Test that we do not process young waiting builds. """
consumer = mock.MagicMock()
@@ -401,7 +403,7 @@ class TestPoller:
# Change the batch to 2, so the module build is in state where
# it is not building anything, but the state is "build".
module_build = models.ModuleBuild.query.filter_by(id=3).one()
module_build.state = 1
module_build.state = models.BUILD_STATES[state]
original = datetime.utcnow() - timedelta(minutes=9)
module_build.time_modified = original
db.session.commit()