modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-06-14 22:17:34 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

In case module is stuck in 'init' state for more than 10 minutes, send fake MBSModule msg.

Browse Source 
This fixes issues when UMB message delivery from frontend to backend fails
for whatever reason...

We do the same thing for 'wait' state already, so this commit just extends
it to 'init' state too.
This commit is contained in:
Jan Kaluza
2018-12-18 08:06:31 +01:00
parent cc303854ad
commit bcfead0809
2 changed files with 24 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
module_build_service/scheduler/producer.py
View File

@@ -194,17 +194,22 @@ class MBSProducer(PollingProducer):
log.info(' * {0} components in batch {1}' log.info(' * {0} components in batch {1}'
.format(n, i)) .format(n, i))
def process_waiting_module_builds(self, session): def _nudge_module_builds_in_state(self, session, state, older_than_minutes):
log.info('Looking for module builds stuck in the wait state') """
builds = models.ModuleBuild.by_state(session, 'wait') Finds all the module builds in the `state` with `time_modified` older
log.info(' {0!r} module builds in the wait state...' than `older_than_minutes` and adds fake MBSModule message to the
.format(len(builds))) work queue.
"""
log.info('Looking for module builds stuck in the %s state', state)
builds = models.ModuleBuild.by_state(session, state)
log.info(' {0!r} module builds in the %s state...'
.format(len(builds), state))
now = datetime.utcnow() now = datetime.utcnow()
ten_minutes = timedelta(minutes=10) time_modified_threshold = timedelta(minutes=older_than_minutes)
for build in builds: for build in builds:
# Only give builds a nudge if stuck for more than ten minutes # Only give builds a nudge if stuck for more than ten minutes
if (now - build.time_modified) < ten_minutes: if (now - build.time_modified) < time_modified_threshold:
continue continue
# Pretend the build is modified, so we don't tight spin. # Pretend the build is modified, so we don't tight spin.
@@ -212,12 +217,16 @@ class MBSProducer(PollingProducer):
session.commit() session.commit()
# Fake a message to kickstart the build anew in the consumer # Fake a message to kickstart the build anew in the consumer
state = module_build_service.models.BUILD_STATES['wait'] state = module_build_service.models.BUILD_STATES[state]
msg = module_build_service.messaging.MBSModule( msg = module_build_service.messaging.MBSModule(
'fake message', build.id, state) 'nudge_module_builds_fake_message', build.id, state)
log.info(" Scheduling faked event %r" % msg) log.info(" Scheduling faked event %r" % msg)
module_build_service.scheduler.consumer.work_queue_put(msg) module_build_service.scheduler.consumer.work_queue_put(msg)
def process_waiting_module_builds(self, session):
for state in ['init', 'wait']:
self._nudge_module_builds_in_state(session, state, 10)
def process_open_component_builds(self, session): def process_open_component_builds(self, session):
log.warning('process_open_component_builds is not yet implemented...') log.warning('process_open_component_builds is not yet implemented...')

10
tests/test_scheduler/test_poller.py
View File

@@ -356,8 +356,9 @@ class TestPoller:
koji_session.deleteBuildTarget.assert_not_called() koji_session.deleteBuildTarget.assert_not_called()
@pytest.mark.parametrize('state', ['init', 'wait'])
def test_process_waiting_module_build( def test_process_waiting_module_build(
self, create_builder, global_consumer, dbg): self, create_builder, global_consumer, dbg, state):
""" Test that processing old waiting module builds works. """ """ Test that processing old waiting module builds works. """
consumer = mock.MagicMock() consumer = mock.MagicMock()
@@ -370,7 +371,7 @@ class TestPoller:
# Change the batch to 2, so the module build is in state where # Change the batch to 2, so the module build is in state where
# it is not building anything, but the state is "build". # it is not building anything, but the state is "build".
module_build = models.ModuleBuild.query.filter_by(id=3).one() module_build = models.ModuleBuild.query.filter_by(id=3).one()
module_build.state = 1 module_build.state = models.BUILD_STATES[state]
original = datetime.utcnow() - timedelta(minutes=11) original = datetime.utcnow() - timedelta(minutes=11)
module_build.time_modified = original module_build.time_modified = original
db.session.commit() db.session.commit()
@@ -387,8 +388,9 @@ class TestPoller:
# ensure the time_modified was changed. # ensure the time_modified was changed.
assert module_build.time_modified > original assert module_build.time_modified > original
@pytest.mark.parametrize('state', ['init', 'wait'])
def test_process_waiting_module_build_not_old_enough( def test_process_waiting_module_build_not_old_enough(
self, create_builder, global_consumer, dbg): self, create_builder, global_consumer, dbg, state):
""" Test that we do not process young waiting builds. """ """ Test that we do not process young waiting builds. """
consumer = mock.MagicMock() consumer = mock.MagicMock()
@@ -401,7 +403,7 @@ class TestPoller:
# Change the batch to 2, so the module build is in state where # Change the batch to 2, so the module build is in state where
# it is not building anything, but the state is "build". # it is not building anything, but the state is "build".
module_build = models.ModuleBuild.query.filter_by(id=3).one() module_build = models.ModuleBuild.query.filter_by(id=3).one()
module_build.state = 1 module_build.state = models.BUILD_STATES[state]
original = datetime.utcnow() - timedelta(minutes=9) original = datetime.utcnow() - timedelta(minutes=9)
module_build.time_modified = original module_build.time_modified = original
db.session.commit() db.session.commit()