modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-14 10:59:45 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Some more state transitions.

Browse Source
This commit is contained in:
Ralph Bean
2016-07-15 23:20:23 -04:00
parent fa91f23be5
commit 0ab9793305
6 changed files with 213 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
rida.py
View File

@@ -1,7 +1,5 @@
#!/usr/bin/python3
# -*- coding: utf-8 -*-
# Copyright (c) 2016 Red Hat, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -30,6 +28,7 @@ This is the implementation of the orchestrator's public RESTful API.
"""
from flask import Flask, request
import flask
import json
import logging
import modulemd
@@ -38,9 +37,9 @@ import rida.auth
import rida.config
import rida.database
import rida.logger
import rida.messaging
import rida.scm
import ssl
import shutil
import tempfile
app = Flask(__name__)
@@ -50,6 +49,8 @@ app.config.from_envvar("RIDA_SETTINGS", silent=True)
conf = rida.config.from_file("rida.conf")
rida.logger.init_logging(conf)
log = logging.getLogger(__name__)
db = rida.database.Database(conf)
@app.route("/rida/module-builds/", methods=["POST"])
@@ -77,13 +78,12 @@ def submit_build():
return "The submitted scmurl isn't allowed", 403
yaml = str()
try:
td = tempfile.mkdtemp()
scm = rida.scm.SCM(url, conf.scmurls)
td = tempfile.TemporaryDirectory()
cod = scm.checkout(td.name)
cod = scm.checkout(td)
cofn = os.path.join(cod, (scm.name + ".yaml"))
with open(cofn, "r") as mmdfile:
yaml = mmdfile.read()
td.cleanup()
except Exception as e:
if "is not in the list of allowed SCMs" in str(e):
rc = 403
@@ -92,6 +92,8 @@ def submit_build():
else:
rc = 500
return str(e), rc
finally:
shutil.rmtree(td)
mmd = modulemd.ModuleMetadata()
try:
mmd.loads(yaml)
@@ -100,25 +102,19 @@ def submit_build():
if db.session.query(rida.database.ModuleBuild).filter_by(name=mmd.name,
version=mmd.version, release=mmd.release).first():
return "Module already exists", 409
module = rida.database.ModuleBuild(
module = rida.database.ModuleBuild.create(
db.session,
conf,
name=mmd.name,
version=mmd.version,
release=mmd.release,
state="init",
modulemd=yaml,
)
db.session.add(module)
db.session.commit()
rida.messaging.publish(
modname='rida',
topic='module.state.change',
msg=module.json(), # Note the state is "init" here...
backend=conf.messaging,
)
def failure(message, code):
# TODO, we should make some note of why it failed in a log...
module.transition(rida.database.BUILD_STATES["failed"])
# TODO, we should make some note of why it failed in the db..
log.exception(message)
module.transition(conf, rida.database.BUILD_STATES["failed"])
db.session.add(module)
db.session.commit()
return message, code
@@ -142,26 +138,18 @@ def submit_build():
build = rida.database.ComponentBuild(module_id=module.id, package=pkgname, format="rpms")
db.session.add(build)
module.modulemd = mmd.dumps()
module.transition(rida.database.BUILD_STATES["wait"])
module.transition(conf, rida.database.BUILD_STATES["wait"])
db.session.add(module)
db.session.commit()
# Publish to whatever bus we're configured to connect to.
# This should notify ridad to start doing the work we just scheduled.
rida.messaging.publish(
modname='rida',
topic='module.state.change',
msg=module.json(), # Note the state is "wait" here...
backend=conf.messaging,
)
logging.info("%s submitted build of %s-%s-%s", username, mmd.name,
mmd.version, mmd.release)
return json.dumps(module.json()), 201
return flask.jsonify(module.json()), 201
@app.route("/rida/module-builds/", methods=["GET"])
def query_builds():
"""Lists all tracked module builds."""
return json.dumps([{"id": x.id, "state": x.state}
return flask.jsonify([{"id": x.id, "state": x.state}
for x in db.session.query(rida.database.ModuleBuild).all()]), 200
@@ -175,7 +163,7 @@ def query_build(id):
for build in db.session.query(rida.database.ComponentBuild).filter_by(module_id=id).all():
tasks[build.format + "/" + build.package] = \
str(build.task) + "/" + build.state
return json.dumps({
return flask.jsonify({
"id": module.id,
"state": module.state,
"tasks": tasks

53
rida/database.py
View File

@@ -38,6 +38,8 @@ from sqlalchemy.orm import (
)
from sqlalchemy.ext.declarative import declarative_base
import rida.messaging
import logging
log = logging.getLogger(__name__)
@@ -129,6 +131,7 @@ class ModuleBuild(Base):
release = Column(String, nullable=False)
state = Column(Integer, nullable=False)
modulemd = Column(String, nullable=False)
koji_tag = Column(String) # This gets set after 'wait'
module = relationship('Module', backref='module_builds', lazy=False)
@@ -146,11 +149,57 @@ class ModuleBuild(Base):
raise ValueError("%r is not a module message." % msg['topic'])
return session.query(cls).filter(cls.id==msg['msg']['id']).one()
def transition(self, state):
@classmethod
def create(cls, session, conf, name, version, release, modulemd):
module = cls(
name=name,
version=version,
release=release,
state="init",
modulemd=modulemd,
)
session.add(module)
session.commit()
rida.messaging.publish(
modname='rida',
topic='module.state.change',
msg=module.json(), # Note the state is "init" here...
backend=conf.messaging,
)
return module
def transition(self, conf, state):
""" Record that a build has transitioned state. """
old_state = self.state
self.state = state
log.debug("%r, state %r->%r" % (old_state, self.state))
log.debug("%r, state %r->%r" % (self, old_state, self.state))
rida.messaging.publish(
modname='rida',
topic='module.state.change',
msg=self.json(), # Note the state is "init" here...
backend=conf.messaging,
)
@classmethod
def get_active_by_koji_tag(cls, session, koji_tag):
""" Find the ModuleBuilds in our database that should be in-flight...
... for a given koji tag.
There should be at most one.
"""
query = session.query(rida.database.ModuleBuild)\
.filter_by(koji_tag=koji_tag)\
.filter_by(state="build")
count = query.count()
if count > 1:
raise RuntimeError("%r module builds in flight for %r" % (count, koji_tag))
elif count == 0:
# No builds in flight scheduled by us. Just ignore this.
return None
# Otherwise, there is exactly one module build - it must be ours.
return query.one()
def json(self):
return {

66
rida/scheduler/handlers/components.py Normal file
View File

@@ -0,0 +1,66 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2016 Red Hat, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Written by Ralph Bean <rbean@redhat.com>
""" Handlers for koji component build events on the message bus. """
import rida.builder
import rida.database
import rida.pdc
import logging
import koji
log = logging.getLogger(__name__)
def complete(config, session, msg):
""" Called whenever a koji build completes. """
# First, find our ModuleBuild associated with this repo, if any.
component_build = rida.database.ComponentBuild.from_fedmsg(session, msg)
if not component_build:
template = "We have no record of {name}-{version}-{release}"
log.debug(template.format(**msg['msg']))
return
# Mark the state in the db.
component_build.state = koji.BUILD_STATES['COMPLETE']
session.commit()
# Find all of the sibling builds of this particular build.
parent = component_build.module_build
siblings = parent.component_builds
# Are any of them still executing?
if any([c.state == koji.BUILD_STATES['BUILDING'] for c in siblings]):
# Then they're not all done yet... continue to wait
return
# Otherwise, check to see if any failed.
if any([c.state != koji.BUILD_STATES['COMPLETE'] for c in siblings]):
# They didn't all succeed.. so mark this module build as a failure.
parent.transition(config, rida.BUILD_STATES['failed'])
session.commit()
return
# Otherwise.. if all of the builds succeeded, then mark the module as good.
parent.transition(config, rida.BUILD_STATES['done'])

33
rida/scheduler/handlers/modules.py
View File

@@ -47,7 +47,13 @@ def wait(config, session, msg):
module_info = build.json()
log.debug("Received module_info=%s from pdc" % module_info)
tag = rida.pdc.get_module_tag(pdc_session, module_info)
log.info("Found tag=%s for module %r" % (tag, build))
log.debug("Found tag=%s for module %r" % (tag, build))
# Hang on to this information for later. We need to know which build is
# associated with which koji tag, so that when their repos are regenerated
# in koji we can figure out which for which module build that event is
# relevant.
build.tag = tag
dependencies = rida.pdc.get_module_dependencies(pdc_session, module_info)
builder = rida.builder.KojiModuleBuilder(build.name, config)
@@ -57,30 +63,7 @@ def wait(config, session, msg):
# TODO submit build from srpm to koji
# TODO: buildroot.add_artifact(build_with_dist_tags)
# TODO: buildroot.ready(artifact=$artifact)
build.transition(state="build") # Wait for the buildroot to be ready.
build.transition(conf, state="build") # Wait for the buildroot to be ready.
session.commit()
def build(config, session, msg):
""" Called whenever a module enters the "build" state.
We usually transition to this state once the buildroot is ready.
All we do here is kick off builds of all our components.
"""
module_build = rida.database.ModuleBuild.from_fedmsg(session, msg)
builder = rida.builder.KojiModuleBuilder(build.name, config)
builder.buildroot_resume()
for component_build in module_build.component_builds:
scmurl = "{dist_git}/rpms/{package}?#{gitref}".format(
dist_git=config.dist_git_url,
package=component_build.package,
gitref=component_build.gitref, # This is the update stream
)
artifact_name = 'TODO'
component_build.task = builder.build(artifact_name, scmurl)
component_build.state = koji.BUILD_STATES['BUILDING']
build.transition(state="build") # Now wait for all of those to finish.
session.commit()

63
rida/scheduler/handlers/repos.py Normal file
View File

@@ -0,0 +1,63 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2016 Red Hat, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Written by Ralph Bean <rbean@redhat.com>
""" Handlers for repo change events on the message bus. """
import rida.builder
import rida.database
import rida.pdc
import logging
import koji
log = logging.getLogger(__name__)
def done(config, session, msg):
""" Called whenever koji rebuilds a repo, any repo. """
# First, find our ModuleBuild associated with this repo, if any.
tag = msg['msg']['tag']
module_build = rida.database.ModuleBuild.get_active_by_koji_tag(
session, koji_tag=tag)
if not module_build:
log.debug("No module build found associated with koji tag %r" % tag)
return
unbuilt_components = (
component_build for component_build in module_build.component_builds
if component_build.state is None
)
builder = rida.builder.KojiModuleBuilder(module_build.name, config)
builder.buildroot_resume()
for component_build in unbuilt_components:
scmurl = "{dist_git}/rpms/{package}?#{gitref}".format(
dist_git=config.dist_git_url,
package=component_build.package,
gitref=component_build.gitref, # This is the update stream
)
artifact_name = 'TODO'
component_build.state = koji.BUILD_STATES['BUILDING']
component_build.task = builder.build(artifact_name, scmurl)
session.commit()

10
rida/scheduler/main.py
View File

@@ -76,8 +76,10 @@ class Messaging(threading.Thread):
koji.BUILD_STATES["BUILDING"]: lambda x: x
}
on_module_change = {
rida.BUILD_STATES["init"]: rida.scheduler.handlers.modules.init,
rida.BUILD_STATES["wait"]: rida.scheduler.handlers.modules.wait,
}
# Only one kind of repo change event...
on_repo_change = rida.scheduler.handlers.repos.done,
def sanity_check(self):
""" On startup, make sure our implementation is sane. """
@@ -107,8 +109,10 @@ class Messaging(threading.Thread):
log.debug(msg)
# Choose a handler for this message
if '.buildsys.build.state.change' in msg['topic']:
handler = self.on_build_change[msg['msg']['init']]
if '.buildsys.repo.done' in msg['topic']:
handler = self.on_repo_change
elif '.buildsys.build.state.change' in msg['topic']:
handler = self.on_build_change[msg['msg']['new']]
elif '.rida.module.state.change' in msg['topic']:
handler = self.on_module_change[module_build_state_from_msg(msg)]
else: