modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-05-11 10:34:30 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Move module build to ready from done according to Greenwave

Browse Source 
Signed-off-by: Chenxiong Qi <cqi@redhat.com>
This commit is contained in:
Chenxiong Qi
2019-03-26 09:50:46 +08:00
parent 3992a1f1ac
commit 0ee801877b
7 changed files with 348 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
module_build_service/builder/KojiModuleBuilder.py
View File

@@ -442,6 +442,15 @@ chmod 644 %buildroot/etc/rpm/macros.zz-modules
@staticmethod
@module_build_service.utils.retry(wait_on=(xmlrpclib.ProtocolError, koji.GenericError))
def get_session(config, login=True):
"""Create and return a koji.ClientSession object
:param config: the config object returned from :meth:`init_config`.
:type config: :class:`Config`
:param bool login: whether to log into the session. To login if True
is passed, otherwise not to log into session.
:return: the Koji session object.
:rtype: :class:`koji.ClientSession`
"""
koji_config = munch.Munch(koji.read_config(
profile_name=config.koji_profile,
user_config=config.koji_config,

7
module_build_service/config.py
View File

@@ -535,6 +535,13 @@ class Config(object):
"redhat-rpm-config", "fedpkg-minimal", "rpm-build", "shadow-utils"],
'desc': ('The list packages for offline module build RPM buildroot.')
},
'greenwave_decision_context': {
'type': str,
'default': 'osci_compose_gate_modules',
'desc': 'The Greenwave decision context that whose messages should '
'be handled by MBS. By default, MBS handles Greenwave '
'messages for OSCI.',
}
}
def __init__(self, conf_section_obj):

29
module_build_service/messaging.py
View File

@@ -107,9 +107,11 @@ class FedmsgMessageParser(MessageParser):
topic_categories = _messaging_backends['fedmsg']['services']
categories_re = '|'.join(map(re.escape, topic_categories))
regex_pattern = re.compile(
(r'(?P<category>' + categories_re + r')(?:(?:\.)'
r'(?P<object>build|repo|module))?(?:(?:\.)'
r'(?P<subobject>state|build))?(?:\.)(?P<event>change|done|end|tag)$'))
r'(?P<category>' + categories_re + r')'
r'(?:(?:\.)(?P<object>build|repo|module|decision))?'
r'(?:(?:\.)(?P<subobject>state|build))?'
r'(?:\.)(?P<event>change|done|end|tag|update)$'
)
regex_results = re.search(regex_pattern, topic)
if regex_results:
@@ -169,6 +171,14 @@ class FedmsgMessageParser(MessageParser):
msg_obj = MBSModule(
msg_id, msg_inner_msg.get('id'), msg_inner_msg.get('state'))
elif (category == 'greenwave' and object == 'decision' and
subobject is None and event == 'update'):
msg_obj = GreenwaveDecisionUpdate(
msg_id=msg_id,
decision_context=msg_inner_msg.get('decision_context'),
policies_satisfied=msg_inner_msg.get('policies_satisfied'),
subject_identifier=msg_inner_msg.get('subject_identifier'))
# If the message matched the regex and is important to the app,
# it will be returned
if msg_obj:
@@ -246,6 +256,17 @@ class MBSModule(BaseMessage):
self.module_build_state = module_build_state
class GreenwaveDecisionUpdate(BaseMessage):
"""A class representing message send to topic greenwave.decision.update"""
def __init__(self, msg_id, decision_context, policies_satisfied,
subject_identifier):
super(GreenwaveDecisionUpdate, self).__init__(msg_id)
self.decision_context = decision_context
self.policies_satisfied = policies_satisfied
self.subject_identifier = subject_identifier
def publish(topic, msg, conf, service):
"""
Publish a single message to a given backend, and return
@@ -316,7 +337,7 @@ def _in_memory_publish(topic, msg, conf, service):
_fedmsg_backend = {
'publish': _fedmsg_publish,
'services': ['buildsys', 'mbs'],
'services': ['buildsys', 'mbs', 'greenwave'],
'parser': FedmsgMessageParser(),
'topic_suffix': '.',
}

26
module_build_service/models.py
View File

@@ -270,6 +270,18 @@ class ModuleBuild(MBSBase):
if component.batch <= self.batch
]
@staticmethod
def get_by_id(session, module_build_id):
"""Find out a module build by id and return
:param session: SQLAlchemy database session object.
:param int module_build_id: the module build id to find out.
:return: the found module build. None is returned if no module build
with specified id in database.
:rtype: :class:`ModuleBuild`
"""
return session.query(ModuleBuild).filter(ModuleBuild.id == module_build_id).first()
@staticmethod
def get_last_build_in_all_streams(session, name):
"""
@@ -541,7 +553,19 @@ class ModuleBuild(MBSBase):
return module
def transition(self, conf, state, state_reason=None):
""" Record that a build has transitioned state. """
"""Record that a build has transitioned state.
The history of state transitions are recorded in model
``ModuleBuildTrace``. If transform to a different state, for example
from ``build`` to ``done``, message will be sent to configured message
bus.
:param conf: MBS config object returned from function :func:`init_config`
which contains loaded configs.
:type conf: :class:`Config`
:param int state: the state value to transition to. Refer to ``BUILD_STATES``.
:param str state_reason: optional reason of why to transform to ``state``.
"""
now = datetime.utcnow()
old_state = self.state
self.state = state

9
module_build_service/scheduler/consumer.py
View File

@@ -41,14 +41,17 @@ import moksha.hub
import six
import sqlalchemy.exc
from module_build_service.utils import module_build_state_from_msg
import module_build_service.messaging
import module_build_service.scheduler.handlers.repos
import module_build_service.scheduler.handlers.components
import module_build_service.scheduler.handlers.modules
import module_build_service.scheduler.handlers.tags
import module_build_service.scheduler.handlers.greenwave
import module_build_service.monitor as monitor
from module_build_service import models, log, conf
from module_build_service.scheduler.handlers import greenwave
from module_build_service.utils import module_build_state_from_msg
class MBSConsumer(fedmsg.consumers.FedmsgConsumer):
@@ -129,6 +132,7 @@ class MBSConsumer(fedmsg.consumers.FedmsgConsumer):
# Only one kind of repo change event, though...
self.on_repo_change = module_build_service.scheduler.handlers.repos.done
self.on_tag_change = module_build_service.scheduler.handlers.tags.tagged
self.on_decision_update = module_build_service.scheduler.handlers.greenwave.decision_update
self.sanity_check()
def shutdown(self):
@@ -232,6 +236,9 @@ class MBSConsumer(fedmsg.consumers.FedmsgConsumer):
elif type(msg) == module_build_service.messaging.MBSModule:
handler = self.on_module_change[module_build_state_from_msg(msg)]
build = models.ModuleBuild.from_module_event(session, msg)
elif type(msg) == module_build_service.messaging.GreenwaveDecisionUpdate:
handler = self.on_decision_update
build = greenwave.get_corresponding_module_build(msg.subject_identifier)
else:
return

95
module_build_service/scheduler/handlers/greenwave.py Normal file
View File

@@ -0,0 +1,95 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019 Red Hat, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Written by Chenxiong Qi <cqi@redhat.com>
from module_build_service import conf, db, log
from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder
from module_build_service.models import ModuleBuild, BUILD_STATES
def get_corresponding_module_build(nvr):
"""Find corresponding module build from database and return
:param str nvr: module build NVR. This is the subject_identifier included
inside ``greenwave.decision.update`` message.
:return: the corresponding module build object. For whatever the reason,
if the original module build id cannot be found from the Koji build of
``nvr``, None will be returned.
:rtype: :class:`ModuleBuild` or None
"""
koji_session = KojiModuleBuilder.get_session(conf, login=False)
build_info = koji_session.getBuild(nvr)
if build_info is None:
return None
try:
module_build_id = build_info['extra']['typeinfo']['module'][
'module_build_service_id']
except KeyError:
# If any of the keys is not present, the NVR is not the one for
# handling Greenwave event.
return None
return ModuleBuild.get_by_id(db.session, module_build_id)
def decision_update(config, session, msg):
"""Move module build to ready or failed according to Greenwave result
:param config: the config object returned from function :func:`init_config`,
which is loaded from configuration file.
:type config: :class:`Config`
:param session: the SQLAlchemy database session object.
:param msg: the message object representing a message received from topic
``greenwave.decision.update``.
:type msg: :class:`GreenwaveDecisionUpdate`
"""
if msg.decision_context != config.greenwave_decision_context:
log.debug('Skip Greenwave message %s as MBS only handles message in '
'decision context %s',
msg.msg_id, msg.decision_context)
return
module_build_nvr = msg.subject_identifier
if not msg.policies_satisfied:
log.debug('Skip to handle module build %s because it has not satisfied'
' Greenwave policies.',
module_build_nvr)
return
build = get_corresponding_module_build(module_build_nvr)
if build is None:
log.debug('No corresponding module build of subject_identifier %s is '
'found.', module_build_nvr)
return
if build.state == BUILD_STATES['done']:
build.transition(
conf, BUILD_STATES['ready'],
state_reason='Module build {} has satisfied Greenwave policies.'
.format(module_build_nvr))
else:
log.warning('Module build %s is not in done state but Greenwave tells '
'it passes tests in decision context %s',
module_build_nvr, msg.decision_context)