From 5563886c07f9dee88c96a2ac734c36c050d50cca Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 13 Jul 2016 12:56:19 -0400 Subject: [PATCH 01/11] basic logging for ridad. --- ridad.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/ridad.py b/ridad.py index ba25355f..49818441 100755 --- a/ridad.py +++ b/ridad.py @@ -1,7 +1,5 @@ #!/usr/bin/python3 # -*- coding: utf-8 -*- - - # Copyright (c) 2016 Red Hat, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -31,11 +29,15 @@ This is the main component of the orchestrator and is responsible for proper scheduling component builds in the supported build systems. """ +import logging import os import threading + import rida.config import rida.messaging +log = logging.getLogger() + # TODO: Load the config file from environment config = rida.config.from_file("rida.conf") @@ -74,11 +76,15 @@ class Polling(threading.Thread): # TODO: Sleep for a configuration-determined interval pass -try: - messaging_thread = Messaging() - polling_thread = Polling() - messaging_thread.start() - polling_thread.start() -except KeyboardInterrupt: - # FIXME: Make this less brutal - os._exit() + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) # For now + logging.info("Starting ridad.") + try: + messaging_thread = Messaging() + polling_thread = Polling() + messaging_thread.start() + polling_thread.start() + except KeyboardInterrupt: + # FIXME: Make this less brutal + os._exit() From b5bd9d4cb1344b6eb2b0ef81678802fdb358443a Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 13 Jul 2016 12:56:29 -0400 Subject: [PATCH 02/11] No need for the extra loop here. --- ridad.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/ridad.py b/ridad.py index 49818441..7969d8d6 100755 --- a/ridad.py +++ b/ridad.py @@ -48,22 +48,22 @@ config = rida.config.from_file("rida.conf") class Messaging(threading.Thread): def run(self): - while True: - # TODO: Listen for bus messages from rida about module builds - # entering the wait state - # TODO: Listen for bus messages from the buildsystem about - # component builds changing state - # TODO: Check for modules that can be set to done/failed - # TODO: Act on these things somehow - # TODO: Emit messages about doing so - for msg in rida.messaging.listen(backend=config.messaging): - print("Saw %r with %r" % (msg['topic'], msg)) - if '.buildsys.build.state.change' in msg['topic']: - print("A build changed state in koji!!") - elif '.rida.module.state.change' in msg['topic']: - print("Our frontend says that a module changed state!!") - else: - pass + # TODO: Listen for bus messages from rida about module builds + # entering the wait state + # TODO: Listen for bus messages from the buildsystem about + # component builds changing state + # TODO: Check for modules that can be set to done/failed + # TODO: Act on these things somehow + # TODO: Emit messages about doing so + for msg in rida.messaging.listen(backend=config.messaging): + log.debug("Saw %r, %r" % (msg['msg_id'], msg['topic'])) + if '.buildsys.build.state.change' in msg['topic']: + log.info("A build changed state in koji!!") + elif '.rida.module.state.change' in msg['topic']: + log.info("Our frontend says that a module changed state!!") + else: + pass + class Polling(threading.Thread): def run(self): From bd30153febf4fda2e1f2872e6fd3142b0e6d84bd Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 13 Jul 2016 14:03:17 -0400 Subject: [PATCH 03/11] Move ridad into an importable location and validate module state strings. --- rida.py | 2 +- rida/database.py | 36 ++++++++++++++++++++++++++---- ridad.py => rida/scheduler/main.py | 7 +++--- 3 files changed, 36 insertions(+), 9 deletions(-) rename ridad.py => rida/scheduler/main.py (95%) diff --git a/rida.py b/rida.py index ea7c2aa6..3646ba5f 100755 --- a/rida.py +++ b/rida.py @@ -123,7 +123,7 @@ def submit_build(): build = rida.database.Build(module_id=module.id, package=pkgname, format="rpms") db.session.add(build) module.modulemd = mmd.dumps() - module.state = "wait" + module.state = rida.database.BUILD_STATES["init"] db.session.add(module) db.session.commit() # Publish to whatever bus we're configured to connect to. diff --git a/rida/database.py b/rida/database.py index ac89a529..207d490c 100644 --- a/rida/database.py +++ b/rida/database.py @@ -24,11 +24,32 @@ """Database handler functions.""" -from sqlalchemy import Column, Integer, String, ForeignKey, create_engine -from sqlalchemy.orm import sessionmaker, relationship +from sqlalchemy import ( + Column, + Integer, + String, + ForeignKey, + create_engine, +) +from sqlalchemy.orm import ( + sessionmaker, + relationship, + validates, +) from sqlalchemy.ext.declarative import declarative_base +# Just like koji.BUILD_STATES, except our own codes for modules. +BUILD_STATES = { + "init": 0, + "wait": 1, + "build": 2, + "done": 3, + "failed": 4, + "ready": 5, +} + + class RidaBase(object): # TODO -- we can implement functionality here common to all our model # classes. @@ -77,10 +98,17 @@ class Module(Base): name = Column(String, nullable=False) version = Column(String, nullable=False) release = Column(String, nullable=False) - # XXX: Consider making this a proper ENUM - state = Column(String, nullable=False) + state = Column(Integer, nullable=False) modulemd = Column(String, nullable=False) + @validates('state') + def validate_state(self, key, field): + if field in BUILD_STATES.values(): + return field + if field in BUILD_STATES: + return BUILD_STATES[field] + raise ValueError("%s: %s, not in %r" % (key, field, BUILD_STATES)) + def json(self): return { 'id': self.id, diff --git a/ridad.py b/rida/scheduler/main.py similarity index 95% rename from ridad.py rename to rida/scheduler/main.py index 7969d8d6..96294669 100755 --- a/ridad.py +++ b/rida/scheduler/main.py @@ -58,13 +58,12 @@ class Messaging(threading.Thread): for msg in rida.messaging.listen(backend=config.messaging): log.debug("Saw %r, %r" % (msg['msg_id'], msg['topic'])) if '.buildsys.build.state.change' in msg['topic']: - log.info("A build changed state in koji!!") + self.handle_build_change(msg) elif '.rida.module.state.change' in msg['topic']: - log.info("Our frontend says that a module changed state!!") + self.handle_module_change(msg) else: pass - class Polling(threading.Thread): def run(self): while True: @@ -77,7 +76,7 @@ class Polling(threading.Thread): pass -if __name__ == '__main__': +def main(): logging.basicConfig(level=logging.DEBUG) # For now logging.info("Starting ridad.") try: From 12a9f6d7dbc69ae574313951765508712bcc2ad2 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 13 Jul 2016 14:39:00 -0400 Subject: [PATCH 04/11] Give the Database a context manager interface. --- rida.py | 2 +- rida/database.py | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/rida.py b/rida.py index 3646ba5f..51178c81 100755 --- a/rida.py +++ b/rida.py @@ -50,7 +50,7 @@ app.config.from_envvar("RIDA_SETTINGS", silent=True) conf = rida.config.from_file("rida.conf") rida.logger.init_logging(conf) -db = rida.database.Database() +db = rida.database.Database(conf) @app.route("/rida/module-builds/", methods=["POST"]) def submit_build(): diff --git a/rida/database.py b/rida/database.py index 207d490c..6b602a5f 100644 --- a/rida/database.py +++ b/rida/database.py @@ -62,17 +62,24 @@ Base = declarative_base(cls=RidaBase) class Database(object): """Class for handling database connections.""" - def __init__(self, rdburl=None, debug=False): + def __init__(self, config, debug=False): """Initialize the database object.""" - if not isinstance(rdburl, str): - rdburl = "sqlite:///rida.db" - engine = create_engine(rdburl, echo=debug) - Session = sessionmaker(bind=engine) - self._session = Session() + self.engine = create_engine(config.db, echo=debug) + self._session = None # Lazilly created.. + + def __enter__(self): + return self.session() + + def __exit__(self, *args, **kwargs): + self._session.close() + self._session = None @property def session(self): """Database session object.""" + if not self._session: + Session = sessionmaker(bind=self.engine) + self._session = Session() return self._session @classmethod From e644f0b9df39d32d9fe1523ad4a734ac909c77cd Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 13 Jul 2016 14:39:23 -0400 Subject: [PATCH 05/11] Some structure for event handling in ridad.py. --- rida/scheduler/main.py | 46 +++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 7 deletions(-) mode change 100755 => 100644 rida/scheduler/main.py diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py old mode 100755 new mode 100644 index 96294669..57c4bc52 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -29,6 +29,8 @@ This is the main component of the orchestrator and is responsible for proper scheduling component builds in the supported build systems. """ + +import inspect import logging import os import threading @@ -36,6 +38,8 @@ import threading import rida.config import rida.messaging +import koji + log = logging.getLogger() # TODO: Load the config file from environment @@ -47,22 +51,50 @@ config = rida.config.from_file("rida.conf") # TODO: Set the build state to failed if the module build fails. class Messaging(threading.Thread): + on_build_change = { + koji.BUILD_STATES["BUILDING"]: lambda x: x + } + on_module_change = { + rida.BUILD_STATES["new"]: rida.scheduler.handlers.modules.new, + } + def sanity_check(self): + """ On startup, make sure our implementation is sane. """ + # Ensure we have every state covered + for state in rida.BUILD_STATES: + if state not in self.on_module_change: + raise KeyError("Module build states %r not handled." % state) + for state in koji.BUILD_STATES: + if state not in self.on_build_change: + raise KeyError("Koji build states %r not handled." % state) + + all_fns = self.on_build_change.items() + self.on_module_change.items() + for key, callback in all_fns: + expected = ['conf', 'db', 'msg'] + argspec = inspect.getargspec(callback) + if argspec != expected: + raise ValueError("Callback %r, state %r has argspec %r!=%r" % ( + callback, key, argspec, expected)) + def run(self): - # TODO: Listen for bus messages from rida about module builds - # entering the wait state - # TODO: Listen for bus messages from the buildsystem about - # component builds changing state + self.sanity_check() # TODO: Check for modules that can be set to done/failed # TODO: Act on these things somehow # TODO: Emit messages about doing so for msg in rida.messaging.listen(backend=config.messaging): log.debug("Saw %r, %r" % (msg['msg_id'], msg['topic'])) + + # Choose a handler for this message if '.buildsys.build.state.change' in msg['topic']: - self.handle_build_change(msg) + handler = self.on_build_change[msg['msg']['new']] elif '.rida.module.state.change' in msg['topic']: - self.handle_module_change(msg) + handler = self.on_module_change[msg['msg']['state']] else: - pass + log.debug("Unhandled message...") + continue + + # Execute our chosen handler + with rida.Database(config) as session: + handler(config, session, msg) class Polling(threading.Thread): def run(self): From e470b4f0da0729fb71d406e076542e9799ae902f Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 13 Jul 2016 14:39:46 -0400 Subject: [PATCH 06/11] Bring back a main entrypoint for ridad.py. --- rida/scheduler/__init__.py | 0 ridad.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 rida/scheduler/__init__.py create mode 100755 ridad.py diff --git a/rida/scheduler/__init__.py b/rida/scheduler/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ridad.py b/ridad.py new file mode 100755 index 00000000..96ba083d --- /dev/null +++ b/ridad.py @@ -0,0 +1,30 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Petr Šabata +# Ralph Bean +"""The module build orchestrator for Modularity, the builder. """ + +import rida.scheduler.main + +if __name__ == '__main__': + rida.scheduler.main.main() From 14a453658316aaf3573ad97a6a0fe2fd196c8470 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 13 Jul 2016 15:40:39 -0400 Subject: [PATCH 07/11] Reorganize the db somewhat.. --- createdb.py | 2 +- rida.py | 12 ++++++------ rida/__init__.py | 1 + rida/database.py | 31 ++++++++++++++++++------------- rida/scheduler/main.py | 6 ++++++ 5 files changed, 32 insertions(+), 20 deletions(-) diff --git a/createdb.py b/createdb.py index 6d97d47a..8b65d919 100755 --- a/createdb.py +++ b/createdb.py @@ -5,4 +5,4 @@ import rida.database config = rida.config.from_file("rida.conf") -rida.database.Database.create_tables(config.db, True) +rida.database.Database.create_tables(config, True) diff --git a/rida.py b/rida.py index 51178c81..8b067fd5 100755 --- a/rida.py +++ b/rida.py @@ -97,10 +97,10 @@ def submit_build(): mmd.loads(yaml) except: return "Invalid modulemd", 422 - if db.session.query(rida.database.Module).filter_by(name=mmd.name, + if db.session.query(rida.database.ModuleBuild).filter_by(name=mmd.name, version=mmd.version, release=mmd.release).first(): return "Module already exists", 409 - module = rida.database.Module(name=mmd.name, version=mmd.version, + module = rida.database.ModuleBuild(name=mmd.name, version=mmd.version, release=mmd.release, state="init", modulemd=yaml) db.session.add(module) db.session.commit() @@ -120,7 +120,7 @@ def submit_build(): return "Failed to get the latest commit: %s" % pkgname, 422 if not rida.scm.SCM(pkg["repository"] + "?#" + pkg["commit"]).is_available(): return "Cannot checkout %s" % pkgname, 422 - build = rida.database.Build(module_id=module.id, package=pkgname, format="rpms") + build = rida.database.ComponentBuild(module_id=module.id, package=pkgname, format="rpms") db.session.add(build) module.modulemd = mmd.dumps() module.state = rida.database.BUILD_STATES["init"] @@ -143,17 +143,17 @@ def submit_build(): def query_builds(): """Lists all tracked module builds.""" return json.dumps([{"id": x.id, "state": x.state} - for x in db.session.query(rida.database.Module).all()]), 200 + for x in db.session.query(rida.database.ModuleBuild).all()]), 200 @app.route("/rida/module-builds/", methods=["GET"]) def query_build(id): """Lists details for the specified module builds.""" - module = db.session.query(rida.database.Module).filter_by(id=id).first() + module = db.session.query(rida.database.ModuleBuild).filter_by(id=id).first() if module: tasks = dict() if module.state != "init": - for build in db.session.query(rida.database.Build).filter_by(module_id=id).all(): + for build in db.session.query(rida.database.ComponentBuild).filter_by(module_id=id).all(): tasks[build.format + "/" + build.package] = \ str(build.task) + "/" + build.state return json.dumps({ diff --git a/rida/__init__.py b/rida/__init__.py index e6b6df0d..dc2fc0a8 100644 --- a/rida/__init__.py +++ b/rida/__init__.py @@ -41,3 +41,4 @@ for a number of tasks: infrastructure services can pick up the work. """ +from rida.database import BUILD_STATES diff --git a/rida/database.py b/rida/database.py index 6b602a5f..cdceaa34 100644 --- a/rida/database.py +++ b/rida/database.py @@ -83,31 +83,36 @@ class Database(object): return self._session @classmethod - def create_tables(cls, db_url, debug=False): + def create_tables(cls, config, debug=False): """ Creates our tables in the database. - :arg db_url, URL used to connect to the database. The URL contains - information with regards to the database engine, the host to connect - to, the user and password and the database name. + :arg config, config object with a 'db' URL attached to it. ie: ://:@/ :kwarg debug, a boolean specifying wether we should have the verbose output of sqlalchemy or not. :return a Database connection that can be used to query to db. """ - engine = create_engine(db_url, echo=debug) + engine = create_engine(config.db, echo=debug) Base.metadata.create_all(engine) - return cls(db_url, debug=debug) + return cls(config, debug=debug) class Module(Base): __tablename__ = "modules" + name = Column(String, primary_key=True) + + +class ModuleBuild(Base): + __tablename__ = "module_builds" id = Column(Integer, primary_key=True) - name = Column(String, nullable=False) + name = Column(String, ForeignKey('modules.name'), nullable=False) version = Column(String, nullable=False) release = Column(String, nullable=False) state = Column(Integer, nullable=False) modulemd = Column(String, nullable=False) + module = relationship('Module', backref='module_builds', lazy=False) + @validates('state') def validate_state(self, key, field): if field in BUILD_STATES.values(): @@ -128,12 +133,12 @@ class Module(Base): #'modulemd': self.modulemd, # TODO, show their entire .json() ? - 'builds': [build.id for build in self.builds], + 'component_builds': [build.id for build in self.component_builds], } -class Build(Base): - __tablename__ = "builds" +class ComponentBuild(Base): + __tablename__ = "component_builds" id = Column(Integer, primary_key=True) package = Column(String, nullable=False) # XXX: Consider making this a proper ENUM @@ -142,8 +147,8 @@ class Build(Base): # XXX: Consider making this a proper ENUM (or an int) state = Column(String) - module_id = Column(Integer, ForeignKey('modules.id'), nullable=False) - module = relationship('Module', backref='builds', lazy=False) + module_id = Column(Integer, ForeignKey('module_builds.id'), nullable=False) + module_build = relationship('ModuleBuild', backref='component_builds', lazy=False) def json(self): return { @@ -152,5 +157,5 @@ class Build(Base): 'format': self.format, 'task': self.task, 'state': self.state, - 'module': self.module.id, + 'module_build': self.module_build.id, } diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py index 57c4bc52..bc7cf87c 100644 --- a/rida/scheduler/main.py +++ b/rida/scheduler/main.py @@ -37,6 +37,8 @@ import threading import rida.config import rida.messaging +import rida.scheduler.handlers.modules +#import rida.scheduler.handlers.builds import koji @@ -51,12 +53,16 @@ config = rida.config.from_file("rida.conf") # TODO: Set the build state to failed if the module build fails. class Messaging(threading.Thread): + + # These are our main lookup tables for figuring out what to run in response + # to what messaging events. on_build_change = { koji.BUILD_STATES["BUILDING"]: lambda x: x } on_module_change = { rida.BUILD_STATES["new"]: rida.scheduler.handlers.modules.new, } + def sanity_check(self): """ On startup, make sure our implementation is sane. """ # Ensure we have every state covered From ff831df01d62e10498e8edd95f964a64904bccc8 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 13 Jul 2016 16:55:34 -0400 Subject: [PATCH 08/11] Adjust this name. --- rida/builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rida/builder.py b/rida/builder.py index 2217ace1..3d267df5 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -136,11 +136,11 @@ class Builder: """ if backend == "koji": - return KojiModuleBuild(module=module, config=config) + return KojiModuleBuilder(module=module, config=config) else: raise ValueError("Builder backend='%s' not recognized" % backend) -class KojiModuleBuild(GenericBuilder): +class KojiModuleBuilder(GenericBuilder): """ Koji specific builder class """ backend = "koji" From 0859641cb8ab28e838cb43397030c5b91b70830e Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 13 Jul 2016 16:55:41 -0400 Subject: [PATCH 09/11] A useful utility classmethod. --- rida/database.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rida/database.py b/rida/database.py index cdceaa34..62706968 100644 --- a/rida/database.py +++ b/rida/database.py @@ -121,6 +121,12 @@ class ModuleBuild(Base): return BUILD_STATES[field] raise ValueError("%s: %s, not in %r" % (key, field, BUILD_STATES)) + @classmethod + def from_fedmsg(cls, session, msg): + if '.module.' not in msg['topic']: + raise ValueError("%r is not a module message." % msg['topic']) + return session.query(cls).filter_by(cls.id==msg['msg']['id']) + def json(self): return { 'id': self.id, From 40a3448dcac1c873765af2da568c2473f630fba4 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 13 Jul 2016 16:55:51 -0400 Subject: [PATCH 10/11] Some handlers and one test. --- rida/scheduler/handlers/__init__.py | 0 rida/scheduler/handlers/modules.py | 67 +++++++++++++++++++ test-requirements.txt | 2 + tests/__init__.py | 0 tests/test_scheduler/__init__.py | 0 tests/test_scheduler/test_modules/__init__.py | 0 .../test_scheduler/test_modules/test_init.py | 55 +++++++++++++++ 7 files changed, 124 insertions(+) create mode 100644 rida/scheduler/handlers/__init__.py create mode 100644 rida/scheduler/handlers/modules.py create mode 100644 test-requirements.txt create mode 100644 tests/__init__.py create mode 100644 tests/test_scheduler/__init__.py create mode 100644 tests/test_scheduler/test_modules/__init__.py create mode 100644 tests/test_scheduler/test_modules/test_init.py diff --git a/rida/scheduler/handlers/__init__.py b/rida/scheduler/handlers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py new file mode 100644 index 00000000..8df6342e --- /dev/null +++ b/rida/scheduler/handlers/modules.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Ralph Bean + +""" Handlers for module change events on the message bus. """ + +import rida.builder +import rida.database +import rida.pdc + + +def init(config, session, msg): + """ Called whenever a module enters the 'init' state. + + We usually transition to this state when the modulebuild is first requested. + + All we do here is request preparation of the buildroot. + """ + build = rida.database.ModuleBuild.from_fedmsg(session, msg) + pdc = rida.pdc.get_pdc_client_session(config) + module_info = build.to_pdc_module_info() + tag = rida.pdc.get_module_tag(pdc, module_info) + dependencies = rida.pdc.get_module_dependencies(pdc, module_info) + builder = rida.builder.KojiModuleBuilder(build.name, config) + builder.buildroot_add_dependency(dependencies) + build.buildroot_task_id = builder.buildroot_prep() + build.state = "wait" # Wait for the buildroot to be ready. + + +def build(config, session, msg): + """ Called whenever a module enters the "build" state. + + We usually transition to this state once the buildroot is ready. + + All we do here is kick off builds of all our components. + """ + module_build = rida.database.ModuleBuild.from_fedmsg(session, msg) + for component_build in module_build.component_builds: + scmurl = "{dist_git}/rpms/{package}?#{gitref}".format( + dist_git=config.dist_git_url, + package=component_build.package, + gitref=component_build.gitref, # This is the update stream + ) + artifact_name = 'TODO' + component_build.task = builder.build(artifact_name, scmurl) + component_build.state = koji.BUILD_STATES['BUILDING'] + + build.state = "build" # Now wait for all of those to finish. diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 00000000..a6786964 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,2 @@ +nose +mock diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_scheduler/__init__.py b/tests/test_scheduler/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_scheduler/test_modules/__init__.py b/tests/test_scheduler/test_modules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_scheduler/test_modules/test_init.py b/tests/test_scheduler/test_modules/test_init.py new file mode 100644 index 00000000..294e1be8 --- /dev/null +++ b/tests/test_scheduler/test_modules/test_init.py @@ -0,0 +1,55 @@ +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Ralph Bean + +import unittest +import mock + +import rida.scheduler.handlers.modules + + +class TestInit(unittest.TestCase): + + def setUp(self): + self.config = mock.Mock() + self.session = mock.Mock() + self.fn = rida.scheduler.handlers.modules.init + + @mock.patch('rida.builder.KojiModuleBuilder') + @mock.patch('rida.database.ModuleBuild.from_fedmsg') + @mock.patch('rida.pdc.get_pdc_client_session') + def test_init_basic(self, pdc, from_fedmsg, KojiModuleBuilder): + builder = mock.Mock() + KojiModuleBuilder.return_value = builder + mocked_module_build = mock.Mock() + mocked_module_build.to_pdc_module_info.return_value = { + 'name': 'foo', + 'version': 1, + } + from_fedmsg.return_value = mocked_module_build + + msg = { + 'topic': 'org.fedoraproject.prod.rida.module.state.change', + 'msg': { + 'id': 1, + }, + } + self.fn(config=self.config, session=self.session, msg=msg) From 371c23d6bba8d45c5e450c11c5268eef797cdc28 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Jul 2016 11:23:32 -0400 Subject: [PATCH 11/11] Descriptions of the states. --- README.rst | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ rida.py | 2 +- rida/database.py | 16 +++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 96ebb84d..6ceb3ed2 100644 --- a/README.rst +++ b/README.rst @@ -131,3 +131,54 @@ Possible response codes are for various requests include: - HTTP 501 Not Implemented - The requested URL is valid but the handler isn't implemented yet. - HTTP 503 Service Unavailable - The service is down, possibly for maintanance. + +Module Build States +------------------- + +You can see the list of possible states with:: + + import rida + print(rida.BUILD_STATES) + +Here's a description of what each of them means: + +init +~~~~ + +This is (obviously) the first state a module build enters. + +When a user first submits a module build, it enters this state. We parse the +modulemd file, learn the NVR, and create a record for the module build. + +Then, we validate that the components are available, and that we can fetch +them. If this is all good, then we set the build to the 'wait' state. If +anything goes wrong, we jump immediately to the 'failed' state. + +wait +~~~~ + +Here, the scheduler picks up tasks in wait and switches to build immediately. +Eventually, we'll add throttling logic here so we don't submit too many builds for the build system to handle. + +build +~~~~~ + +The scheduler works on builds in this state. We prepare the buildroot, submit +builds for all the components, and wait for the results to come back. + +done +~~~~ + +Once all components have succeeded, we set the top-level module build to 'done'. + +failed +~~~~~~ + +If any of the component builds fail, then we set the top-level module build to 'failed' also. + +ready +~~~~~ + +This is a state to be set when a module is ready to be part of a +larger compose. perhaps it is set by an external service that knows +about the Grand Plan. diff --git a/rida.py b/rida.py index 8b067fd5..f5a322b4 100755 --- a/rida.py +++ b/rida.py @@ -123,7 +123,7 @@ def submit_build(): build = rida.database.ComponentBuild(module_id=module.id, package=pkgname, format="rpms") db.session.add(build) module.modulemd = mmd.dumps() - module.state = rida.database.BUILD_STATES["init"] + module.state = rida.database.BUILD_STATES["wait"] db.session.add(module) db.session.commit() # Publish to whatever bus we're configured to connect to. diff --git a/rida/database.py b/rida/database.py index 62706968..26c60da0 100644 --- a/rida/database.py +++ b/rida/database.py @@ -41,11 +41,27 @@ from sqlalchemy.ext.declarative import declarative_base # Just like koji.BUILD_STATES, except our own codes for modules. BUILD_STATES = { + # When you parse the modulemd file and know the nvr and you create a + # record in the db, and that's it. + # publish the message + # validate that components are available + # and that you can fetch them. + # if all is good, go to wait: telling ridad to take over. + # if something is bad, go straight to failed. "init": 0, + # Here, the scheduler picks up tasks in wait. + # switch to build immediately. + # throttling logic (when we write it) goes here. "wait": 1, + # Actively working on it. "build": 2, + # All is good "done": 3, + # Something failed "failed": 4, + # This is a state to be set when a module is ready to be part of a + # larger compose. perhaps it is set by an external service that knows + # about the Grand Plan. "ready": 5, }