Merge #20 Work on ridad.

README.rst

@@ -131,3 +131,54 @@ Possible response codes are for various requests include:
 - HTTP 501 Not Implemented - The requested URL is valid but the handler isn't
   implemented yet.
 - HTTP 503 Service Unavailable - The service is down, possibly for maintanance.
+
+Module Build States
+-------------------
+
+You can see the list of possible states with::
+
+    import rida
+    print(rida.BUILD_STATES)
+
+Here's a description of what each of them means:
+
+init
+~~~~
+
+This is (obviously) the first state a module build enters.
+
+When a user first submits a module build, it enters this state.  We parse the
+modulemd file, learn the NVR, and create a record for the module build.
+
+Then, we validate that the components are available, and that we can fetch
+them.  If this is all good, then we set the build to the 'wait' state.  If
+anything goes wrong, we jump immediately to the 'failed' state.
+
+wait
+~~~~
+
+Here, the scheduler picks up tasks in wait and switches to build immediately.
+Eventually, we'll add throttling logic here so we don't submit too many builds for the build system to handle.
+
+build
+~~~~~
+
+The scheduler works on builds in this state.  We prepare the buildroot, submit
+builds for all the components, and wait for the results to come back.
+
+done
+~~~~
+
+Once all components have succeeded, we set the top-level module build to 'done'.
+
+failed
+~~~~~~
+
+If any of the component builds fail, then we set the top-level module build to 'failed' also.
+
+ready
+~~~~~
+
+This is a state to be set when a module is ready to be part of a
+larger compose.  perhaps it is set by an external service that knows
+about the Grand Plan.

createdb.py

@@ -5,4 +5,4 @@ import rida.database
 
 config = rida.config.from_file("rida.conf")
 
-rida.database.Database.create_tables(config.db, True)
+rida.database.Database.create_tables(config, True)

rida.py

@@ -50,7 +50,7 @@ app.config.from_envvar("RIDA_SETTINGS", silent=True)
 conf = rida.config.from_file("rida.conf")
 rida.logger.init_logging(conf)
 
-db = rida.database.Database()
+db = rida.database.Database(conf)
 
 @app.route("/rida/module-builds/", methods=["POST"])
 def submit_build():
@@ -97,10 +97,10 @@ def submit_build():
         mmd.loads(yaml)
     except:
         return "Invalid modulemd", 422
-    if db.session.query(rida.database.Module).filter_by(name=mmd.name,
+    if db.session.query(rida.database.ModuleBuild).filter_by(name=mmd.name,
         version=mmd.version, release=mmd.release).first():
         return "Module already exists", 409
-    module = rida.database.Module(name=mmd.name, version=mmd.version,
+    module = rida.database.ModuleBuild(name=mmd.name, version=mmd.version,
             release=mmd.release, state="init", modulemd=yaml)
     db.session.add(module)
     db.session.commit()
@@ -120,10 +120,10 @@ def submit_build():
                 return "Failed to get the latest commit: %s" % pkgname, 422
         if not rida.scm.SCM(pkg["repository"] + "?#" + pkg["commit"]).is_available():
             return "Cannot checkout %s" % pkgname, 422
-        build = rida.database.Build(module_id=module.id, package=pkgname, format="rpms")
+        build = rida.database.ComponentBuild(module_id=module.id, package=pkgname, format="rpms")
         db.session.add(build)
     module.modulemd = mmd.dumps()
-    module.state = "wait"
+    module.state = rida.database.BUILD_STATES["wait"]
     db.session.add(module)
     db.session.commit()
     # Publish to whatever bus we're configured to connect to.
@@ -143,17 +143,17 @@ def submit_build():
 def query_builds():
     """Lists all tracked module builds."""
     return json.dumps([{"id": x.id, "state": x.state}
-        for x in db.session.query(rida.database.Module).all()]), 200
+        for x in db.session.query(rida.database.ModuleBuild).all()]), 200
 
 
 @app.route("/rida/module-builds/<int:id>", methods=["GET"])
 def query_build(id):
     """Lists details for the specified module builds."""
-    module = db.session.query(rida.database.Module).filter_by(id=id).first()
+    module = db.session.query(rida.database.ModuleBuild).filter_by(id=id).first()
     if module:
         tasks = dict()
         if module.state != "init":
-            for build in db.session.query(rida.database.Build).filter_by(module_id=id).all():
+            for build in db.session.query(rida.database.ComponentBuild).filter_by(module_id=id).all():
                 tasks[build.format + "/" + build.package] = \
                     str(build.task) + "/" + build.state
         return json.dumps({

rida/__init__.py

@@ -41,3 +41,4 @@ for a number of tasks:
   infrastructure services can pick up the work.
 """
 
+from rida.database import BUILD_STATES

rida/builder.py

@@ -136,11 +136,11 @@ class Builder:
         """
 
         if backend == "koji":
-            return KojiModuleBuild(module=module, config=config)
+            return KojiModuleBuilder(module=module, config=config)
         else:
             raise ValueError("Builder backend='%s' not recognized" % backend)
 
-class KojiModuleBuild(GenericBuilder):
+class KojiModuleBuilder(GenericBuilder):
     """ Koji specific builder class """
 
     backend = "koji"

rida/database.py

@@ -24,11 +24,48 @@
 
 """Database handler functions."""
 
-from sqlalchemy import Column, Integer, String, ForeignKey, create_engine
-from sqlalchemy.orm import sessionmaker, relationship
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    ForeignKey,
+    create_engine,
+)
+from sqlalchemy.orm import (
+    sessionmaker,
+    relationship,
+    validates,
+)
 from sqlalchemy.ext.declarative import declarative_base
 
 
+# Just like koji.BUILD_STATES, except our own codes for modules.
+BUILD_STATES = {
+    # When you parse the modulemd file and know the nvr and you create a
+    # record in the db, and that's it.
+    # publish the message
+    # validate that components are available
+    #   and that you can fetch them.
+    # if all is good, go to wait: telling ridad to take over.
+    # if something is bad, go straight to failed.
+    "init": 0,
+    # Here, the scheduler picks up tasks in wait.
+    # switch to build immediately.
+    # throttling logic (when we write it) goes here.
+    "wait": 1,
+    # Actively working on it.
+    "build": 2,
+    # All is good
+    "done": 3,
+    # Something failed
+    "failed": 4,
+    # This is a state to be set when a module is ready to be part of a
+    # larger compose.  perhaps it is set by an external service that knows
+    # about the Grand Plan.
+    "ready": 5,
+}
+
+
 class RidaBase(object):
     # TODO -- we can implement functionality here common to all our model
     # classes.
@@ -41,46 +78,71 @@ Base = declarative_base(cls=RidaBase)
 class Database(object):
     """Class for handling database connections."""
 
-    def __init__(self, rdburl=None, debug=False):
+    def __init__(self, config, debug=False):
         """Initialize the database object."""
-        if not isinstance(rdburl, str):
-            rdburl = "sqlite:///rida.db"
-        engine = create_engine(rdburl, echo=debug)
-        Session = sessionmaker(bind=engine)
-        self._session = Session()
+        self.engine = create_engine(config.db, echo=debug)
+        self._session = None  # Lazilly created..
+
+    def __enter__(self):
+        return self.session()
+
+    def __exit__(self, *args, **kwargs):
+        self._session.close()
+        self._session = None
 
     @property
     def session(self):
         """Database session object."""
+        if not self._session:
+            Session = sessionmaker(bind=self.engine)
+            self._session = Session()
         return self._session
 
     @classmethod
-    def create_tables(cls, db_url, debug=False):
+    def create_tables(cls, config, debug=False):
         """ Creates our tables in the database.
 
-        :arg db_url, URL used to connect to the database. The URL contains
-        information with regards to the database engine, the host to connect
-        to, the user and password and the database name.
+        :arg config, config object with a 'db' URL attached to it.
         ie: <engine>://<user>:<password>@<host>/<dbname>
         :kwarg debug, a boolean specifying wether we should have the verbose
         output of sqlalchemy or not.
         :return a Database connection that can be used to query to db.
         """
-        engine = create_engine(db_url, echo=debug)
+        engine = create_engine(config.db, echo=debug)
         Base.metadata.create_all(engine)
-        return cls(db_url, debug=debug)
+        return cls(config, debug=debug)
 
 
 class Module(Base):
     __tablename__ = "modules"
+    name = Column(String, primary_key=True)
+
+
+class ModuleBuild(Base):
+    __tablename__ = "module_builds"
     id = Column(Integer, primary_key=True)
-    name = Column(String, nullable=False)
+    name = Column(String, ForeignKey('modules.name'), nullable=False)
     version = Column(String, nullable=False)
     release = Column(String, nullable=False)
-    # XXX: Consider making this a proper ENUM
-    state = Column(String, nullable=False)
+    state = Column(Integer, nullable=False)
     modulemd = Column(String, nullable=False)
 
+    module = relationship('Module', backref='module_builds', lazy=False)
+
+    @validates('state')
+    def validate_state(self, key, field):
+        if field in BUILD_STATES.values():
+            return field
+        if field in BUILD_STATES:
+            return BUILD_STATES[field]
+        raise ValueError("%s: %s, not in %r" % (key, field, BUILD_STATES))
+
+    @classmethod
+    def from_fedmsg(cls, session, msg):
+        if '.module.' not in msg['topic']:
+            raise ValueError("%r is not a module message." % msg['topic'])
+        return session.query(cls).filter_by(cls.id==msg['msg']['id'])
+
     def json(self):
         return {
             'id': self.id,
@@ -93,12 +155,12 @@ class Module(Base):
             #'modulemd': self.modulemd,
 
             # TODO, show their entire .json() ?
-            'builds': [build.id for build in self.builds],
+            'component_builds': [build.id for build in self.component_builds],
         }
 
 
-class Build(Base):
-    __tablename__ = "builds"
+class ComponentBuild(Base):
+    __tablename__ = "component_builds"
     id = Column(Integer, primary_key=True)
     package = Column(String, nullable=False)
     # XXX: Consider making this a proper ENUM
@@ -107,8 +169,8 @@ class Build(Base):
     # XXX: Consider making this a proper ENUM (or an int)
     state = Column(String)
 
-    module_id = Column(Integer, ForeignKey('modules.id'), nullable=False)
-    module = relationship('Module', backref='builds', lazy=False)
+    module_id = Column(Integer, ForeignKey('module_builds.id'), nullable=False)
+    module_build = relationship('ModuleBuild', backref='component_builds', lazy=False)
 
     def json(self):
         return {
@@ -117,5 +179,5 @@ class Build(Base):
             'format': self.format,
             'task': self.task,
             'state': self.state,
-            'module': self.module.id,
+            'module_build': self.module_build.id,
         }

rida/scheduler/handlers/modules.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2016  Red Hat, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+# Written by Ralph Bean <rbean@redhat.com>
+
+""" Handlers for module change events on the message bus. """
+
+import rida.builder
+import rida.database
+import rida.pdc
+
+
+def init(config, session, msg):
+    """ Called whenever a module enters the 'init' state.
+
+    We usually transition to this state when the modulebuild is first requested.
+
+    All we do here is request preparation of the buildroot.
+    """
+    build = rida.database.ModuleBuild.from_fedmsg(session, msg)
+    pdc = rida.pdc.get_pdc_client_session(config)
+    module_info = build.to_pdc_module_info()
+    tag = rida.pdc.get_module_tag(pdc, module_info)
+    dependencies = rida.pdc.get_module_dependencies(pdc, module_info)
+    builder = rida.builder.KojiModuleBuilder(build.name, config)
+    builder.buildroot_add_dependency(dependencies)
+    build.buildroot_task_id = builder.buildroot_prep()
+    build.state = "wait"  # Wait for the buildroot to be ready.
+
+
+def build(config, session, msg):
+    """ Called whenever a module enters the "build" state.
+
+    We usually transition to this state once the buildroot is ready.
+
+    All we do here is kick off builds of all our components.
+    """
+    module_build = rida.database.ModuleBuild.from_fedmsg(session, msg)
+    for component_build in module_build.component_builds:
+        scmurl = "{dist_git}/rpms/{package}?#{gitref}".format(
+            dist_git=config.dist_git_url,
+            package=component_build.package,
+            gitref=component_build.gitref,  # This is the update stream
+        )
+        artifact_name = 'TODO'
+        component_build.task = builder.build(artifact_name, scmurl)
+        component_build.state = koji.BUILD_STATES['BUILDING']
+
+    build.state = "build"  # Now wait for all of those to finish.

rida/scheduler/main.py

@@ -0,0 +1,127 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# Copyright (c) 2016  Red Hat, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+# Written by Petr Šabata <contyk@redhat.com>
+#            Ralph Bean <rbean@redhat.com>
+
+"""The module build orchestrator for Modularity, the builder.
+
+This is the main component of the orchestrator and is responsible for
+proper scheduling component builds in the supported build systems.
+"""
+
+
+import inspect
+import logging
+import os
+import threading
+
+import rida.config
+import rida.messaging
+import rida.scheduler.handlers.modules
+#import rida.scheduler.handlers.builds
+
+import koji
+
+log = logging.getLogger()
+
+# TODO: Load the config file from environment
+config = rida.config.from_file("rida.conf")
+
+# TODO: Utilized rida.builder to prepare the buildroots and build components.
+# TODO: Set the build state to build once the module build is started.
+# TODO: Set the build state to done once the module build is done.
+# TODO: Set the build state to failed if the module build fails.
+
+class Messaging(threading.Thread):
+
+    # These are our main lookup tables for figuring out what to run in response
+    # to what messaging events.
+    on_build_change = {
+        koji.BUILD_STATES["BUILDING"]: lambda x: x
+    }
+    on_module_change = {
+        rida.BUILD_STATES["new"]: rida.scheduler.handlers.modules.new,
+    }
+
+    def sanity_check(self):
+        """ On startup, make sure our implementation is sane. """
+        # Ensure we have every state covered
+        for state in rida.BUILD_STATES:
+            if state not in self.on_module_change:
+                raise KeyError("Module build states %r not handled." % state)
+        for state in koji.BUILD_STATES:
+            if state not in self.on_build_change:
+                raise KeyError("Koji build states %r not handled." % state)
+
+        all_fns = self.on_build_change.items() + self.on_module_change.items()
+        for key, callback in all_fns:
+            expected = ['conf', 'db', 'msg']
+            argspec = inspect.getargspec(callback)
+            if argspec != expected:
+                raise ValueError("Callback %r, state %r has argspec %r!=%r" % (
+                    callback, key, argspec, expected))
+
+    def run(self):
+        self.sanity_check()
+        # TODO: Check for modules that can be set to done/failed
+        # TODO: Act on these things somehow
+        # TODO: Emit messages about doing so
+        for msg in rida.messaging.listen(backend=config.messaging):
+            log.debug("Saw %r, %r" % (msg['msg_id'], msg['topic']))
+
+            # Choose a handler for this message
+            if '.buildsys.build.state.change' in msg['topic']:
+                handler = self.on_build_change[msg['msg']['new']]
+            elif '.rida.module.state.change' in msg['topic']:
+                handler = self.on_module_change[msg['msg']['state']]
+            else:
+                log.debug("Unhandled message...")
+                continue
+
+            # Execute our chosen handler
+            with rida.Database(config) as session:
+                handler(config, session, msg)
+
+class Polling(threading.Thread):
+    def run(self):
+        while True:
+            # TODO: Check for module builds in the wait state
+            # TODO: Check component builds in the open state
+            # TODO: Check for modules that can be set to done/failed
+            # TODO: Act on these things somehow
+            # TODO: Emit messages about doing so
+            # TODO: Sleep for a configuration-determined interval
+            pass
+
+
+def main():
+    logging.basicConfig(level=logging.DEBUG)  # For now
+    logging.info("Starting ridad.")
+    try:
+        messaging_thread = Messaging()
+        polling_thread = Polling()
+        messaging_thread.start()
+        polling_thread.start()
+    except KeyboardInterrupt:
+        # FIXME: Make this less brutal
+        os._exit()

ridad.py

@@ -1,7 +1,5 @@
 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
-
-
 # Copyright (c) 2016  Red Hat, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -24,61 +22,9 @@
 #
 # Written by Petr Šabata <contyk@redhat.com>
 #            Ralph Bean <rbean@redhat.com>
+"""The module build orchestrator for Modularity, the builder. """
 
-"""The module build orchestrator for Modularity, the builder.
+import rida.scheduler.main
 
-This is the main component of the orchestrator and is responsible for
-proper scheduling component builds in the supported build systems.
-"""
-
-import os
-import threading
-import rida.config
-import rida.messaging
-
-# TODO: Load the config file from environment
-config = rida.config.from_file("rida.conf")
-
-# TODO: Utilized rida.builder to prepare the buildroots and build components.
-# TODO: Set the build state to build once the module build is started.
-# TODO: Set the build state to done once the module build is done.
-# TODO: Set the build state to failed if the module build fails.
-
-class Messaging(threading.Thread):
-    def run(self):
-        while True:
-            # TODO: Listen for bus messages from rida about module builds
-            #       entering the wait state
-            # TODO: Listen for bus messages from the buildsystem about
-            #       component builds changing state
-            # TODO: Check for modules that can be set to done/failed
-            # TODO: Act on these things somehow
-            # TODO: Emit messages about doing so
-            for msg in rida.messaging.listen(backend=config.messaging):
-                print("Saw %r with %r" % (msg['topic'], msg))
-                if '.buildsys.build.state.change' in msg['topic']:
-                    print("A build changed state in koji!!")
-                elif '.rida.module.state.change' in msg['topic']:
-                    print("Our frontend says that a module changed state!!")
-                else:
-                    pass
-
-class Polling(threading.Thread):
-    def run(self):
-        while True:
-            # TODO: Check for module builds in the wait state
-            # TODO: Check component builds in the open state
-            # TODO: Check for modules that can be set to done/failed
-            # TODO: Act on these things somehow
-            # TODO: Emit messages about doing so
-            # TODO: Sleep for a configuration-determined interval
-            pass
-
-try:
-    messaging_thread = Messaging()
-    polling_thread = Polling()
-    messaging_thread.start()
-    polling_thread.start()
-except KeyboardInterrupt:
-    # FIXME: Make this less brutal
-    os._exit()
+if __name__ == '__main__':
+    rida.scheduler.main.main()

test-requirements.txt

@@ -0,0 +1,2 @@
+nose
+mock

tests/test_scheduler/test_modules/test_init.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2016  Red Hat, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+# Written by Ralph Bean <rbean@redhat.com>
+
+import unittest
+import mock
+
+import rida.scheduler.handlers.modules
+
+
+class TestInit(unittest.TestCase):
+
+    def setUp(self):
+        self.config = mock.Mock()
+        self.session = mock.Mock()
+        self.fn = rida.scheduler.handlers.modules.init
+
+    @mock.patch('rida.builder.KojiModuleBuilder')
+    @mock.patch('rida.database.ModuleBuild.from_fedmsg')
+    @mock.patch('rida.pdc.get_pdc_client_session')
+    def test_init_basic(self, pdc, from_fedmsg, KojiModuleBuilder):
+        builder = mock.Mock()
+        KojiModuleBuilder.return_value = builder
+        mocked_module_build = mock.Mock()
+        mocked_module_build.to_pdc_module_info.return_value = {
+            'name': 'foo',
+            'version': 1,
+        }
+        from_fedmsg.return_value = mocked_module_build
+
+        msg = {
+            'topic': 'org.fedoraproject.prod.rida.module.state.change',
+            'msg': {
+                'id': 1,
+            },
+        }
+        self.fn(config=self.config, session=self.session, msg=msg)