diff --git a/README.rst b/README.rst index 96ebb84d..6ceb3ed2 100644 --- a/README.rst +++ b/README.rst @@ -131,3 +131,54 @@ Possible response codes are for various requests include: - HTTP 501 Not Implemented - The requested URL is valid but the handler isn't implemented yet. - HTTP 503 Service Unavailable - The service is down, possibly for maintanance. + +Module Build States +------------------- + +You can see the list of possible states with:: + + import rida + print(rida.BUILD_STATES) + +Here's a description of what each of them means: + +init +~~~~ + +This is (obviously) the first state a module build enters. + +When a user first submits a module build, it enters this state. We parse the +modulemd file, learn the NVR, and create a record for the module build. + +Then, we validate that the components are available, and that we can fetch +them. If this is all good, then we set the build to the 'wait' state. If +anything goes wrong, we jump immediately to the 'failed' state. + +wait +~~~~ + +Here, the scheduler picks up tasks in wait and switches to build immediately. +Eventually, we'll add throttling logic here so we don't submit too many builds for the build system to handle. + +build +~~~~~ + +The scheduler works on builds in this state. We prepare the buildroot, submit +builds for all the components, and wait for the results to come back. + +done +~~~~ + +Once all components have succeeded, we set the top-level module build to 'done'. + +failed +~~~~~~ + +If any of the component builds fail, then we set the top-level module build to 'failed' also. + +ready +~~~~~ + +This is a state to be set when a module is ready to be part of a +larger compose. perhaps it is set by an external service that knows +about the Grand Plan. diff --git a/createdb.py b/createdb.py index 6d97d47a..8b65d919 100755 --- a/createdb.py +++ b/createdb.py @@ -5,4 +5,4 @@ import rida.database config = rida.config.from_file("rida.conf") -rida.database.Database.create_tables(config.db, True) +rida.database.Database.create_tables(config, True) diff --git a/rida.py b/rida.py index 9d196db9..6673079e 100755 --- a/rida.py +++ b/rida.py @@ -50,7 +50,7 @@ app.config.from_envvar("RIDA_SETTINGS", silent=True) conf = rida.config.from_file("rida.conf") rida.logger.init_logging(conf) -db = rida.database.Database() +db = rida.database.Database(conf) @app.route("/rida/module-builds/", methods=["POST"]) def submit_build(): @@ -97,10 +97,10 @@ def submit_build(): mmd.loads(yaml) except: return "Invalid modulemd", 422 - if db.session.query(rida.database.Module).filter_by(name=mmd.name, + if db.session.query(rida.database.ModuleBuild).filter_by(name=mmd.name, version=mmd.version, release=mmd.release).first(): return "Module already exists", 409 - module = rida.database.Module(name=mmd.name, version=mmd.version, + module = rida.database.ModuleBuild(name=mmd.name, version=mmd.version, release=mmd.release, state="init", modulemd=yaml) db.session.add(module) db.session.commit() @@ -120,10 +120,10 @@ def submit_build(): return "Failed to get the latest commit: %s" % pkgname, 422 if not rida.scm.SCM(pkg["repository"] + "?#" + pkg["commit"]).is_available(): return "Cannot checkout %s" % pkgname, 422 - build = rida.database.Build(module_id=module.id, package=pkgname, format="rpms") + build = rida.database.ComponentBuild(module_id=module.id, package=pkgname, format="rpms") db.session.add(build) module.modulemd = mmd.dumps() - module.state = "wait" + module.state = rida.database.BUILD_STATES["wait"] db.session.add(module) db.session.commit() # Publish to whatever bus we're configured to connect to. @@ -143,17 +143,17 @@ def submit_build(): def query_builds(): """Lists all tracked module builds.""" return json.dumps([{"id": x.id, "state": x.state} - for x in db.session.query(rida.database.Module).all()]), 200 + for x in db.session.query(rida.database.ModuleBuild).all()]), 200 @app.route("/rida/module-builds/", methods=["GET"]) def query_build(id): """Lists details for the specified module builds.""" - module = db.session.query(rida.database.Module).filter_by(id=id).first() + module = db.session.query(rida.database.ModuleBuild).filter_by(id=id).first() if module: tasks = dict() if module.state != "init": - for build in db.session.query(rida.database.Build).filter_by(module_id=id).all(): + for build in db.session.query(rida.database.ComponentBuild).filter_by(module_id=id).all(): tasks[build.format + "/" + build.package] = \ str(build.task) + "/" + build.state return json.dumps({ diff --git a/rida/__init__.py b/rida/__init__.py index e6b6df0d..dc2fc0a8 100644 --- a/rida/__init__.py +++ b/rida/__init__.py @@ -41,3 +41,4 @@ for a number of tasks: infrastructure services can pick up the work. """ +from rida.database import BUILD_STATES diff --git a/rida/builder.py b/rida/builder.py index 2217ace1..3d267df5 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -136,11 +136,11 @@ class Builder: """ if backend == "koji": - return KojiModuleBuild(module=module, config=config) + return KojiModuleBuilder(module=module, config=config) else: raise ValueError("Builder backend='%s' not recognized" % backend) -class KojiModuleBuild(GenericBuilder): +class KojiModuleBuilder(GenericBuilder): """ Koji specific builder class """ backend = "koji" diff --git a/rida/database.py b/rida/database.py index ac89a529..26c60da0 100644 --- a/rida/database.py +++ b/rida/database.py @@ -24,11 +24,48 @@ """Database handler functions.""" -from sqlalchemy import Column, Integer, String, ForeignKey, create_engine -from sqlalchemy.orm import sessionmaker, relationship +from sqlalchemy import ( + Column, + Integer, + String, + ForeignKey, + create_engine, +) +from sqlalchemy.orm import ( + sessionmaker, + relationship, + validates, +) from sqlalchemy.ext.declarative import declarative_base +# Just like koji.BUILD_STATES, except our own codes for modules. +BUILD_STATES = { + # When you parse the modulemd file and know the nvr and you create a + # record in the db, and that's it. + # publish the message + # validate that components are available + # and that you can fetch them. + # if all is good, go to wait: telling ridad to take over. + # if something is bad, go straight to failed. + "init": 0, + # Here, the scheduler picks up tasks in wait. + # switch to build immediately. + # throttling logic (when we write it) goes here. + "wait": 1, + # Actively working on it. + "build": 2, + # All is good + "done": 3, + # Something failed + "failed": 4, + # This is a state to be set when a module is ready to be part of a + # larger compose. perhaps it is set by an external service that knows + # about the Grand Plan. + "ready": 5, +} + + class RidaBase(object): # TODO -- we can implement functionality here common to all our model # classes. @@ -41,46 +78,71 @@ Base = declarative_base(cls=RidaBase) class Database(object): """Class for handling database connections.""" - def __init__(self, rdburl=None, debug=False): + def __init__(self, config, debug=False): """Initialize the database object.""" - if not isinstance(rdburl, str): - rdburl = "sqlite:///rida.db" - engine = create_engine(rdburl, echo=debug) - Session = sessionmaker(bind=engine) - self._session = Session() + self.engine = create_engine(config.db, echo=debug) + self._session = None # Lazilly created.. + + def __enter__(self): + return self.session() + + def __exit__(self, *args, **kwargs): + self._session.close() + self._session = None @property def session(self): """Database session object.""" + if not self._session: + Session = sessionmaker(bind=self.engine) + self._session = Session() return self._session @classmethod - def create_tables(cls, db_url, debug=False): + def create_tables(cls, config, debug=False): """ Creates our tables in the database. - :arg db_url, URL used to connect to the database. The URL contains - information with regards to the database engine, the host to connect - to, the user and password and the database name. + :arg config, config object with a 'db' URL attached to it. ie: ://:@/ :kwarg debug, a boolean specifying wether we should have the verbose output of sqlalchemy or not. :return a Database connection that can be used to query to db. """ - engine = create_engine(db_url, echo=debug) + engine = create_engine(config.db, echo=debug) Base.metadata.create_all(engine) - return cls(db_url, debug=debug) + return cls(config, debug=debug) class Module(Base): __tablename__ = "modules" + name = Column(String, primary_key=True) + + +class ModuleBuild(Base): + __tablename__ = "module_builds" id = Column(Integer, primary_key=True) - name = Column(String, nullable=False) + name = Column(String, ForeignKey('modules.name'), nullable=False) version = Column(String, nullable=False) release = Column(String, nullable=False) - # XXX: Consider making this a proper ENUM - state = Column(String, nullable=False) + state = Column(Integer, nullable=False) modulemd = Column(String, nullable=False) + module = relationship('Module', backref='module_builds', lazy=False) + + @validates('state') + def validate_state(self, key, field): + if field in BUILD_STATES.values(): + return field + if field in BUILD_STATES: + return BUILD_STATES[field] + raise ValueError("%s: %s, not in %r" % (key, field, BUILD_STATES)) + + @classmethod + def from_fedmsg(cls, session, msg): + if '.module.' not in msg['topic']: + raise ValueError("%r is not a module message." % msg['topic']) + return session.query(cls).filter_by(cls.id==msg['msg']['id']) + def json(self): return { 'id': self.id, @@ -93,12 +155,12 @@ class Module(Base): #'modulemd': self.modulemd, # TODO, show their entire .json() ? - 'builds': [build.id for build in self.builds], + 'component_builds': [build.id for build in self.component_builds], } -class Build(Base): - __tablename__ = "builds" +class ComponentBuild(Base): + __tablename__ = "component_builds" id = Column(Integer, primary_key=True) package = Column(String, nullable=False) # XXX: Consider making this a proper ENUM @@ -107,8 +169,8 @@ class Build(Base): # XXX: Consider making this a proper ENUM (or an int) state = Column(String) - module_id = Column(Integer, ForeignKey('modules.id'), nullable=False) - module = relationship('Module', backref='builds', lazy=False) + module_id = Column(Integer, ForeignKey('module_builds.id'), nullable=False) + module_build = relationship('ModuleBuild', backref='component_builds', lazy=False) def json(self): return { @@ -117,5 +179,5 @@ class Build(Base): 'format': self.format, 'task': self.task, 'state': self.state, - 'module': self.module.id, + 'module_build': self.module_build.id, } diff --git a/rida/scheduler/__init__.py b/rida/scheduler/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/rida/scheduler/handlers/__init__.py b/rida/scheduler/handlers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/rida/scheduler/handlers/modules.py b/rida/scheduler/handlers/modules.py new file mode 100644 index 00000000..8df6342e --- /dev/null +++ b/rida/scheduler/handlers/modules.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Ralph Bean + +""" Handlers for module change events on the message bus. """ + +import rida.builder +import rida.database +import rida.pdc + + +def init(config, session, msg): + """ Called whenever a module enters the 'init' state. + + We usually transition to this state when the modulebuild is first requested. + + All we do here is request preparation of the buildroot. + """ + build = rida.database.ModuleBuild.from_fedmsg(session, msg) + pdc = rida.pdc.get_pdc_client_session(config) + module_info = build.to_pdc_module_info() + tag = rida.pdc.get_module_tag(pdc, module_info) + dependencies = rida.pdc.get_module_dependencies(pdc, module_info) + builder = rida.builder.KojiModuleBuilder(build.name, config) + builder.buildroot_add_dependency(dependencies) + build.buildroot_task_id = builder.buildroot_prep() + build.state = "wait" # Wait for the buildroot to be ready. + + +def build(config, session, msg): + """ Called whenever a module enters the "build" state. + + We usually transition to this state once the buildroot is ready. + + All we do here is kick off builds of all our components. + """ + module_build = rida.database.ModuleBuild.from_fedmsg(session, msg) + for component_build in module_build.component_builds: + scmurl = "{dist_git}/rpms/{package}?#{gitref}".format( + dist_git=config.dist_git_url, + package=component_build.package, + gitref=component_build.gitref, # This is the update stream + ) + artifact_name = 'TODO' + component_build.task = builder.build(artifact_name, scmurl) + component_build.state = koji.BUILD_STATES['BUILDING'] + + build.state = "build" # Now wait for all of those to finish. diff --git a/rida/scheduler/main.py b/rida/scheduler/main.py new file mode 100644 index 00000000..bc7cf87c --- /dev/null +++ b/rida/scheduler/main.py @@ -0,0 +1,127 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Petr Šabata +# Ralph Bean + +"""The module build orchestrator for Modularity, the builder. + +This is the main component of the orchestrator and is responsible for +proper scheduling component builds in the supported build systems. +""" + + +import inspect +import logging +import os +import threading + +import rida.config +import rida.messaging +import rida.scheduler.handlers.modules +#import rida.scheduler.handlers.builds + +import koji + +log = logging.getLogger() + +# TODO: Load the config file from environment +config = rida.config.from_file("rida.conf") + +# TODO: Utilized rida.builder to prepare the buildroots and build components. +# TODO: Set the build state to build once the module build is started. +# TODO: Set the build state to done once the module build is done. +# TODO: Set the build state to failed if the module build fails. + +class Messaging(threading.Thread): + + # These are our main lookup tables for figuring out what to run in response + # to what messaging events. + on_build_change = { + koji.BUILD_STATES["BUILDING"]: lambda x: x + } + on_module_change = { + rida.BUILD_STATES["new"]: rida.scheduler.handlers.modules.new, + } + + def sanity_check(self): + """ On startup, make sure our implementation is sane. """ + # Ensure we have every state covered + for state in rida.BUILD_STATES: + if state not in self.on_module_change: + raise KeyError("Module build states %r not handled." % state) + for state in koji.BUILD_STATES: + if state not in self.on_build_change: + raise KeyError("Koji build states %r not handled." % state) + + all_fns = self.on_build_change.items() + self.on_module_change.items() + for key, callback in all_fns: + expected = ['conf', 'db', 'msg'] + argspec = inspect.getargspec(callback) + if argspec != expected: + raise ValueError("Callback %r, state %r has argspec %r!=%r" % ( + callback, key, argspec, expected)) + + def run(self): + self.sanity_check() + # TODO: Check for modules that can be set to done/failed + # TODO: Act on these things somehow + # TODO: Emit messages about doing so + for msg in rida.messaging.listen(backend=config.messaging): + log.debug("Saw %r, %r" % (msg['msg_id'], msg['topic'])) + + # Choose a handler for this message + if '.buildsys.build.state.change' in msg['topic']: + handler = self.on_build_change[msg['msg']['new']] + elif '.rida.module.state.change' in msg['topic']: + handler = self.on_module_change[msg['msg']['state']] + else: + log.debug("Unhandled message...") + continue + + # Execute our chosen handler + with rida.Database(config) as session: + handler(config, session, msg) + +class Polling(threading.Thread): + def run(self): + while True: + # TODO: Check for module builds in the wait state + # TODO: Check component builds in the open state + # TODO: Check for modules that can be set to done/failed + # TODO: Act on these things somehow + # TODO: Emit messages about doing so + # TODO: Sleep for a configuration-determined interval + pass + + +def main(): + logging.basicConfig(level=logging.DEBUG) # For now + logging.info("Starting ridad.") + try: + messaging_thread = Messaging() + polling_thread = Polling() + messaging_thread.start() + polling_thread.start() + except KeyboardInterrupt: + # FIXME: Make this less brutal + os._exit() diff --git a/ridad.py b/ridad.py index ba25355f..96ba083d 100755 --- a/ridad.py +++ b/ridad.py @@ -1,7 +1,5 @@ #!/usr/bin/python3 # -*- coding: utf-8 -*- - - # Copyright (c) 2016 Red Hat, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -24,61 +22,9 @@ # # Written by Petr Šabata # Ralph Bean +"""The module build orchestrator for Modularity, the builder. """ -"""The module build orchestrator for Modularity, the builder. +import rida.scheduler.main -This is the main component of the orchestrator and is responsible for -proper scheduling component builds in the supported build systems. -""" - -import os -import threading -import rida.config -import rida.messaging - -# TODO: Load the config file from environment -config = rida.config.from_file("rida.conf") - -# TODO: Utilized rida.builder to prepare the buildroots and build components. -# TODO: Set the build state to build once the module build is started. -# TODO: Set the build state to done once the module build is done. -# TODO: Set the build state to failed if the module build fails. - -class Messaging(threading.Thread): - def run(self): - while True: - # TODO: Listen for bus messages from rida about module builds - # entering the wait state - # TODO: Listen for bus messages from the buildsystem about - # component builds changing state - # TODO: Check for modules that can be set to done/failed - # TODO: Act on these things somehow - # TODO: Emit messages about doing so - for msg in rida.messaging.listen(backend=config.messaging): - print("Saw %r with %r" % (msg['topic'], msg)) - if '.buildsys.build.state.change' in msg['topic']: - print("A build changed state in koji!!") - elif '.rida.module.state.change' in msg['topic']: - print("Our frontend says that a module changed state!!") - else: - pass - -class Polling(threading.Thread): - def run(self): - while True: - # TODO: Check for module builds in the wait state - # TODO: Check component builds in the open state - # TODO: Check for modules that can be set to done/failed - # TODO: Act on these things somehow - # TODO: Emit messages about doing so - # TODO: Sleep for a configuration-determined interval - pass - -try: - messaging_thread = Messaging() - polling_thread = Polling() - messaging_thread.start() - polling_thread.start() -except KeyboardInterrupt: - # FIXME: Make this less brutal - os._exit() +if __name__ == '__main__': + rida.scheduler.main.main() diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 00000000..a6786964 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,2 @@ +nose +mock diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_scheduler/__init__.py b/tests/test_scheduler/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_scheduler/test_modules/__init__.py b/tests/test_scheduler/test_modules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_scheduler/test_modules/test_init.py b/tests/test_scheduler/test_modules/test_init.py new file mode 100644 index 00000000..294e1be8 --- /dev/null +++ b/tests/test_scheduler/test_modules/test_init.py @@ -0,0 +1,55 @@ +# Copyright (c) 2016 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Ralph Bean + +import unittest +import mock + +import rida.scheduler.handlers.modules + + +class TestInit(unittest.TestCase): + + def setUp(self): + self.config = mock.Mock() + self.session = mock.Mock() + self.fn = rida.scheduler.handlers.modules.init + + @mock.patch('rida.builder.KojiModuleBuilder') + @mock.patch('rida.database.ModuleBuild.from_fedmsg') + @mock.patch('rida.pdc.get_pdc_client_session') + def test_init_basic(self, pdc, from_fedmsg, KojiModuleBuilder): + builder = mock.Mock() + KojiModuleBuilder.return_value = builder + mocked_module_build = mock.Mock() + mocked_module_build.to_pdc_module_info.return_value = { + 'name': 'foo', + 'version': 1, + } + from_fedmsg.return_value = mocked_module_build + + msg = { + 'topic': 'org.fedoraproject.prod.rida.module.state.change', + 'msg': { + 'id': 1, + }, + } + self.fn(config=self.config, session=self.session, msg=msg)