modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-01 18:01:40 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add component build throttling

Browse Source
This commit is contained in:
Matt Prahl
2016-11-12 21:01:45 -05:00
parent 67d07023c5
commit 244b77f54c
5 changed files with 73 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
module_build_service/models.py
View File

@@ -122,16 +122,22 @@ class ModuleBuild(RidaBase):
module = db.relationship('Module', backref='module_builds', lazy=False)
def current_batch(self):
def current_batch(self, state=None):
""" Returns all components of this module in the current batch. """
if not self.batch:
raise ValueError("No batch is in progress: %r" % self.batch)
return [
component for component in self.component_builds
if component.batch == self.batch
]
if state:
return [
component for component in self.component_builds
if component.batch == self.batch and component.state == state
]
else:
return [
component for component in self.component_builds
if component.batch == self.batch
]
def mmd(self):
mmd = _modulemd.ModuleMetadata()

16
module_build_service/scheduler/handlers/repos.py
View File

@@ -100,18 +100,24 @@ def done(config, session, msg):
# So now we can either start a new batch if there are still some to build
# or, if everything is built successfully, then we can bless the module as
# complete.
leftover_components = [
unbuilt_components = [
c for c in module_build.component_builds
if (c.state != koji.BUILD_STATES['COMPLETE']
and c.state != koji.BUILD_STATES["FAILED"])
]
if leftover_components:
module_build.batch += 1
if unbuilt_components:
# Increment the build batch when no components are being built and all
# have at least attempted a build (even failures) in the current batch
unbuilt_components_in_batch = [
c for c in module_build.current_batch()
if c.state == koji.BUILD_STATES['BUILDING'] or not c.state
]
if not unbuilt_components_in_batch:
module_build.batch += 1
module_build_service.utils.start_build_batch(
config, module_build, session, builder)
else:
module_build.transition(config, state=models.BUILD_STATES['done'])
session.commit()
# And that's it. :)

20
module_build_service/scheduler/main.py
View File

@@ -38,6 +38,7 @@ import six.moves.queue as queue
import module_build_service.config
import module_build_service.messaging
import module_build_service.utils
import module_build_service.scheduler.handlers.components
import module_build_service.scheduler.handlers.modules
import module_build_service.scheduler.handlers.repos
@@ -190,8 +191,8 @@ class Poller(threading.Thread):
# XXX: detect whether it's really stucked first
# self.process_waiting_module_builds(session)
self.process_open_component_builds(session)
self.process_lingering_module_builds(session)
self.fail_lost_builds(session)
self.process_paused_module_builds(conf, session)
log.info("Polling thread sleeping, %rs" % conf.polling_interval)
time.sleep(conf.polling_interval)
@@ -281,8 +282,21 @@ class Poller(threading.Thread):
def process_open_component_builds(self, session):
log.warning("process_open_component_builds is not yet implemented...")
def process_lingering_module_builds(self, session):
log.warning("process_lingering_module_builds is not yet implemented...")
def process_paused_module_builds(self, config, session):
if module_build_service.utils.at_concurrent_component_threshold(
config, session):
log.debug('Will not attempt to start paused module builds due to '
'the concurrent build threshold being met')
return
# Check to see if module builds that are in build state but don't have
# any component builds being built can be worked on
for module_build in session.query(models.ModuleBuild).filter_by(
state=models.BUILD_STATES['build']).all():
# If there are no components in the build state on the module build,
# then no possible event will start off new component builds
if not module_build.current_batch(koji.BUILD_STATES['BUILDING']):
module_build_service.utils.start_build_batch(
config, module_build, session, config.system)
_work_queue = queue.Queue()

33
module_build_service/utils.py
View File

@@ -30,7 +30,6 @@ import shutil
import tempfile
import os
import modulemd
import time
from module_build_service import log, models
from module_build_service.errors import ValidationError, UnprocessableEntity
from module_build_service import app, conf, db, log
@@ -59,9 +58,28 @@ def retry(timeout=120, interval=30, wait_on=Exception):
return wrapper
def at_concurrent_component_threshold(config, session):
"""
Determines if the number of concurrent component builds has reached
the configured threshold
:param config: Module Build Service configuration object
:param session: SQLAlchemy database session
:return: boolean representing if there are too many concurrent builds at
this time
"""
import koji # Placed here to avoid py2/py3 conflicts...
if config.num_consecutive_builds and config.num_consecutive_builds <= \
session.query(models.ComponentBuild).filter_by(
state=koji.BUILD_STATES['BUILDING']).count():
return True
return False
def start_build_batch(config, module, session, builder, components=None):
""" Starts a round of the build cycle for a module. """
import koji # Placed here to avoid py2/py3 conflicts...
if any([c.state == koji.BUILD_STATES['BUILDING']
@@ -70,11 +88,11 @@ def start_build_batch(config, module, session, builder, components=None):
# The user can either pass in a list of components to 'seed' the batch, or
# if none are provided then we just select everything that hasn't
# successfully built yet.
module.batch += 1
# successfully built yet or isn't currently being built.
unbuilt_components = components or [
c for c in module.component_builds
if (c.state != koji.BUILD_STATES['COMPLETE']
and c.state != koji.BUILD_STATES['BUILDING']
and c.batch == module.batch)
]
@@ -82,7 +100,12 @@ def start_build_batch(config, module, session, builder, components=None):
unbuilt_components))
for c in unbuilt_components:
c.task_id, c.state, c.state_reason, c.nvr = builder.build(artifact_name=c.package, source=c.scmurl)
if at_concurrent_component_threshold(config, session):
log.info('Concurrent build threshold met')
break
c.task_id, c.state, c.state_reason, c.nvr = builder.build(
artifact_name=c.package, source=c.scmurl)
if not c.task_id:
module.transition(config, models.BUILD_STATES["failed"],

8
tests/test_scheduler/test_repo_done.py
View File

@@ -54,7 +54,9 @@ class TestRepoDone(unittest.TestCase):
@mock.patch('module_build_service.builder.KojiModuleBuilder.build')
@mock.patch('module_build_service.builder.KojiModuleBuilder.buildroot_connect')
@mock.patch('module_build_service.models.ModuleBuild.from_repo_done_event')
def test_a_single_match(self, from_repo_done_event, connect, build_fn, config, ready):
@mock.patch('module_build_service.utils.at_concurrent_component_threshold',
return_value=False)
def test_a_single_match(self, threshold, from_repo_done_event, connect, build_fn, config, ready):
""" Test that when a repo msg hits us and we have a single match.
"""
config.return_value = mock.Mock(), "development"
@@ -88,7 +90,9 @@ class TestRepoDone(unittest.TestCase):
@mock.patch('module_build_service.builder.KojiModuleBuilder.build')
@mock.patch('module_build_service.builder.KojiModuleBuilder.buildroot_connect')
@mock.patch('module_build_service.models.ModuleBuild.from_repo_done_event')
def test_a_single_match_build_fail(self, from_repo_done_event, connect, build_fn, config, ready):
@mock.patch('module_build_service.utils.at_concurrent_component_threshold',
return_value=False)
def test_a_single_match_build_fail(self, threshold, from_repo_done_event, connect, build_fn, config, ready):
""" Test that when a KojiModuleBuilder.build fails, the build is
marked as failed with proper state_reason.
"""