modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-08 13:18:04 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge #184 Add Build Rate Limiting

Browse Source
This commit is contained in:
Matt Prahl
2016-11-16 20:51:07 +00:00
parent a45eba41e8 244b77f54c
commit efd5d203b3
11 changed files with 98 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Vagrantfile vendored
View File

@@ -27,6 +27,7 @@ $script = <<SCRIPT
ln -s /opt/module_build_service/krb5-stg.fp.o /etc/krb5.conf.d/stg_fedoraproject_org
python manage.py upgradedb
python manage.py generatelocalhostcert
systemctl enable fedmsg-relay
systemctl start fedmsg-relay
echo "export KRB5CCNAME=FILE:/var/tmp/krbcc" >> ~/.bashrc
SCRIPT

4
config.py
View File

@@ -28,6 +28,10 @@ class BaseConfiguration(object):
# Set to zero to disable polling
POLLING_INTERVAL = 600
# Determines how many builds that can be submitted to the builder
# and be in the build state at a time. Set this to 0 for no restrictions
NUM_CONSECUTIVE_BUILDS = 5
RPMS_DEFAULT_REPOSITORY = 'git://pkgs.fedoraproject.org/rpms/'
RPMS_ALLOW_REPOSITORY = False
RPMS_DEFAULT_CACHE = 'http://pkgs.fedoraproject.org/repo/pkgs/'

11
module_build_service/config.py
View File

@@ -204,6 +204,10 @@ class Config(object):
'type': list,
'default': [],
'desc': 'Allowed SCM URLs.'},
'num_consecutive_builds': {
'type': int,
'default': 0,
'desc': 'Number of consecutive component builds.'},
}
def __init__(self):
@@ -305,3 +309,10 @@ class Config(object):
if not isinstance(l, list):
raise TypeError("scmurls needs to be a list.")
self.scmurls = [str(x) for x in l]
def _setifok_num_consecutive_builds(self, i):
if not isinstance(i, int):
raise TypeError('NUM_CONSECUTIVE_BUILDS needs to be an int')
if i < 0:
raise ValueError('NUM_CONSECUTIVE_BUILDS must be >= 0')
self.num_consecutive_builds = i

16
module_build_service/models.py
View File

@@ -122,16 +122,22 @@ class ModuleBuild(RidaBase):
module = db.relationship('Module', backref='module_builds', lazy=False)
def current_batch(self):
def current_batch(self, state=None):
""" Returns all components of this module in the current batch. """
if not self.batch:
raise ValueError("No batch is in progress: %r" % self.batch)
return [
component for component in self.component_builds
if component.batch == self.batch
]
if state:
return [
component for component in self.component_builds
if component.batch == self.batch and component.state == state
]
else:
return [
component for component in self.component_builds
if component.batch == self.batch
]
def mmd(self):
mmd = _modulemd.ModuleMetadata()

18
module_build_service/scheduler/handlers/components.py
View File

@@ -87,24 +87,6 @@ def _finalize(config, session, msg, state):
builder.buildroot_add_artifacts([nvr,], install=install)
session.commit()
# Find all of the sibling builds of this particular build.
current_batch = parent.current_batch()
# If there are no components, transition module build state to 'done'
if not current_batch:
log.info("Batch is empty. Moving module state to DONE.")
parent.transition(config, module.BUILD_STATES['done'])
session.commit()
return
# Otherwise, check to see if all failed. If so, then we cannot continue on
# to a next batch. This module build is doomed.
if all([c.state != koji.BUILD_STATES['COMPLETE'] for c in current_batch]):
# They didn't all succeed.. so mark this module build as a failure.
parent.transition(config, models.BUILD_STATES['failed'], state_reason)
session.commit()
return
def complete(config, session, msg):
return _finalize(config, session, msg, state=koji.BUILD_STATES['COMPLETE'])

5
module_build_service/scheduler/handlers/modules.py
View File

@@ -160,4 +160,7 @@ def wait(config, session, msg):
# If this build already exists and is done, then fake the repo change event
# back to the scheduler
if state == koji.BUILD_STATES['COMPLETE']:
return [module_build_service.messaging.KojiRepoChange('fake msg', build.koji_tag)]
# TODO: builder.module_build_tag only works for Koji, figure out if
# other backends need this implemented (e.g. COPR)
return [module_build_service.messaging.KojiRepoChange(
'fake msg', builder.module_build_tag['name'])]

17
module_build_service/scheduler/handlers/repos.py
View File

@@ -100,17 +100,24 @@ def done(config, session, msg):
# So now we can either start a new batch if there are still some to build
# or, if everything is built successfully, then we can bless the module as
# complete.
leftover_components = [
unbuilt_components = [
c for c in module_build.component_builds
if (c.state != koji.BUILD_STATES['COMPLETE']
and c.state != koji.BUILD_STATES["FAILED"])
]
if leftover_components:
module_build_service.utils.start_next_build_batch(
if unbuilt_components:
# Increment the build batch when no components are being built and all
# have at least attempted a build (even failures) in the current batch
unbuilt_components_in_batch = [
c for c in module_build.current_batch()
if c.state == koji.BUILD_STATES['BUILDING'] or not c.state
]
if not unbuilt_components_in_batch:
module_build.batch += 1
module_build_service.utils.start_build_batch(
config, module_build, session, builder)
else:
module_build.transition(config, state=models.BUILD_STATES['done'])
session.commit()
# And that's it. :)

20
module_build_service/scheduler/main.py
View File

@@ -38,6 +38,7 @@ import six.moves.queue as queue
import module_build_service.config
import module_build_service.messaging
import module_build_service.utils
import module_build_service.scheduler.handlers.components
import module_build_service.scheduler.handlers.modules
import module_build_service.scheduler.handlers.repos
@@ -190,8 +191,8 @@ class Poller(threading.Thread):
# XXX: detect whether it's really stucked first
# self.process_waiting_module_builds(session)
self.process_open_component_builds(session)
self.process_lingering_module_builds(session)
self.fail_lost_builds(session)
self.process_paused_module_builds(conf, session)
log.info("Polling thread sleeping, %rs" % conf.polling_interval)
time.sleep(conf.polling_interval)
@@ -281,8 +282,21 @@ class Poller(threading.Thread):
def process_open_component_builds(self, session):
log.warning("process_open_component_builds is not yet implemented...")
def process_lingering_module_builds(self, session):
log.warning("process_lingering_module_builds is not yet implemented...")
def process_paused_module_builds(self, config, session):
if module_build_service.utils.at_concurrent_component_threshold(
config, session):
log.debug('Will not attempt to start paused module builds due to '
'the concurrent build threshold being met')
return
# Check to see if module builds that are in build state but don't have
# any component builds being built can be worked on
for module_build in session.query(models.ModuleBuild).filter_by(
state=models.BUILD_STATES['build']).all():
# If there are no components in the build state on the module build,
# then no possible event will start off new component builds
if not module_build.current_batch(koji.BUILD_STATES['BUILDING']):
module_build_service.utils.start_build_batch(
config, module_build, session, config.system)
_work_queue = queue.Queue()

36
module_build_service/utils.py
View File

@@ -58,22 +58,41 @@ def retry(timeout=120, interval=30, wait_on=Exception):
return wrapper
def start_next_build_batch(config, module, session, builder, components=None):
""" Starts a next round of the build cycle for a module. """
def at_concurrent_component_threshold(config, session):
"""
Determines if the number of concurrent component builds has reached
the configured threshold
:param config: Module Build Service configuration object
:param session: SQLAlchemy database session
:return: boolean representing if there are too many concurrent builds at
this time
"""
import koji # Placed here to avoid py2/py3 conflicts...
if config.num_consecutive_builds and config.num_consecutive_builds <= \
session.query(models.ComponentBuild).filter_by(
state=koji.BUILD_STATES['BUILDING']).count():
return True
return False
def start_build_batch(config, module, session, builder, components=None):
""" Starts a round of the build cycle for a module. """
import koji # Placed here to avoid py2/py3 conflicts...
if any([c.state == koji.BUILD_STATES['BUILDING']
for c in module.component_builds ]):
raise ValueError("Cannot start a batch when another is in flight.")
# The user can either pass in a list of components to 'seed' the batch, or
# if none are provided then we just select everything that hasn't
# successfully built yet.
module.batch += 1
# successfully built yet or isn't currently being built.
unbuilt_components = components or [
c for c in module.component_builds
if (c.state != koji.BUILD_STATES['COMPLETE']
and c.state != koji.BUILD_STATES['BUILDING']
and c.batch == module.batch)
]
@@ -81,7 +100,12 @@ def start_next_build_batch(config, module, session, builder, components=None):
unbuilt_components))
for c in unbuilt_components:
c.task_id, c.state, c.state_reason, c.nvr = builder.build(artifact_name=c.package, source=c.scmurl)
if at_concurrent_component_threshold(config, session):
log.info('Concurrent build threshold met')
break
c.task_id, c.state, c.state_reason, c.nvr = builder.build(
artifact_name=c.package, source=c.scmurl)
if not c.task_id:
module.transition(config, models.BUILD_STATES["failed"],
@@ -291,6 +315,8 @@ def submit_module_build(username, url):
err_msgs = pool.map(lambda data: "Cannot checkout {}".format(data[0])
if not module_build_service.scm.SCM(data[1]).is_available()
else None, full_urls)
# TODO: only the first error message is raised, perhaps concatenate
# the messages together?
for err_msg in err_msgs:
if err_msg:
raise UnprocessableEntity(err_msg)

1
tests/test_scheduler/test_module_wait.py
View File

@@ -40,6 +40,7 @@ class TestModuleWait(unittest.TestCase):
builder = mock.Mock()
builder.get_disttag_srpm.return_value = 'some srpm disttag'
builder.build.return_value = 1234, 1, "", None
builder.module_build_tag = {'name': 'some-tag-build'}
KojiModuleBuilder.return_value = builder
mocked_module_build = mock.Mock()
mocked_module_build.json.return_value = {

8
tests/test_scheduler/test_repo_done.py
View File

@@ -54,7 +54,9 @@ class TestRepoDone(unittest.TestCase):
@mock.patch('module_build_service.builder.KojiModuleBuilder.build')
@mock.patch('module_build_service.builder.KojiModuleBuilder.buildroot_connect')
@mock.patch('module_build_service.models.ModuleBuild.from_repo_done_event')
def test_a_single_match(self, from_repo_done_event, connect, build_fn, config, ready):
@mock.patch('module_build_service.utils.at_concurrent_component_threshold',
return_value=False)
def test_a_single_match(self, threshold, from_repo_done_event, connect, build_fn, config, ready):
""" Test that when a repo msg hits us and we have a single match.
"""
config.return_value = mock.Mock(), "development"
@@ -88,7 +90,9 @@ class TestRepoDone(unittest.TestCase):
@mock.patch('module_build_service.builder.KojiModuleBuilder.build')
@mock.patch('module_build_service.builder.KojiModuleBuilder.buildroot_connect')
@mock.patch('module_build_service.models.ModuleBuild.from_repo_done_event')
def test_a_single_match_build_fail(self, from_repo_done_event, connect, build_fn, config, ready):
@mock.patch('module_build_service.utils.at_concurrent_component_threshold',
return_value=False)
def test_a_single_match_build_fail(self, threshold, from_repo_done_event, connect, build_fn, config, ready):
""" Test that when a KojiModuleBuilder.build fails, the build is
marked as failed with proper state_reason.
"""