Attempt to recursively build component like mockchain --recurse

rida/database.py

@@ -138,8 +138,24 @@ class ModuleBuild(Base):
     koji_tag = Column(String)  # This gets set after 'wait'
     scmurl = Column(String)
 
+    # A monotonically increasing integer that represents which batch or
+    # iteration this module is currently on for successive rebuilds of its
+    # components.  Think like 'mockchain --recurse'
+    batch = Column(Integer, default=0)
+
     module = relationship('Module', backref='module_builds', lazy=False)
 
+    def current_batch(self):
+        """ Returns all components of this module in the current batch. """
+
+        if not self.batch:
+            raise ValueError("No batch is in progress: %r" % self.batch)
+
+        return [
+            component for component in self.component_builds
+            if component.batch == self.batch
+        ]
+
     def mmd(self):
         mmd = _modulemd.ModuleMetadata()
         try:
@@ -232,9 +248,9 @@ class ModuleBuild(Base):
         }
 
     def __repr__(self):
-        return "<ModuleBuild %s-%s-%s, state %r>" % (
+        return "<ModuleBuild %s-%s-%s, state %r, batch %r>" % (
             self.name, self.version, self.release,
-            INVERSE_BUILD_STATES[self.state])
+            INVERSE_BUILD_STATES[self.state], self.batch)
 
 
 class ComponentBuild(Base):
@@ -248,6 +264,12 @@ class ComponentBuild(Base):
     # XXX: Consider making this a proper ENUM (or an int)
     state = Column(Integer)
 
+    # A monotonically increasing integer that represents which batch or
+    # iteration this *component* is currently in.  This relates to the owning
+    # module's batch.  This one defaults to None, which means that this
+    # component is not currently part of a batch.
+    batch = Column(Integer, default=0)
+
     module_id = Column(Integer, ForeignKey('module_builds.id'), nullable=False)
     module_build = relationship('ModuleBuild', backref='component_builds', lazy=False)
 
@@ -280,5 +302,5 @@ class ComponentBuild(Base):
 
 
     def __repr__(self):
-        return "<ComponentBuild %s of %r, state: %r, task_id: %r>" % (
-            self.package, self.module_id, self.state, self.task_id)
+        return "<ComponentBuild %s, %r, state: %r, task_id: %r, batch: %r>" % (
+            self.package, self.module_id, self.state, self.task_id, self.batch)

rida/scheduler/handlers/components.py

@@ -59,25 +59,16 @@ def _finalize(config, session, msg, state):
 
     # Find all of the sibling builds of this particular build.
     parent = component_build.module_build
-    siblings = parent.component_builds
+    current_batch = parent.current_batch()
 
-    # Are any of them still executing?
-    premature = (koji.BUILD_STATES['BUILDING'], None)
-    if any([c.state in premature for c in siblings]):
-        # Then they're not all done yet... continue to wait
-        return
-
-    # Otherwise, check to see if any failed.
-    if any([c.state != koji.BUILD_STATES['COMPLETE'] for c in siblings]):
+    # Otherwise, check to see if all failed.  If so, then we cannot continue on
+    # to a next batch.  This module build is doomed.
+    if all([c.state != koji.BUILD_STATES['COMPLETE'] for c in current_batch]):
         # They didn't all succeed.. so mark this module build as a failure.
         parent.transition(config, rida.BUILD_STATES['failed'])
         session.commit()
         return
 
-    # Otherwise.. if all of the builds succeeded, then mark the module as good.
-    parent.transition(config, rida.BUILD_STATES['done'])
-    session.commit()
-
 
 def complete(config, session, msg):
     return _finalize(config, session, msg, state=koji.BUILD_STATES['COMPLETE'])

rida/scheduler/handlers/repos.py

@@ -36,6 +36,11 @@ log = logging.getLogger(__name__)
 def done(config, session, msg):
     """ Called whenever koji rebuilds a repo, any repo. """
 
+    # TODO -- working here.
+    #   finished the utilities for getting batch and starting next.
+    #   finished the simpler component-build event
+    #   need to write this stuff, the more complicated bits.
+
     # First, find our ModuleBuild associated with this repo, if any.
     tag = msg['msg']['tag'].strip('-build')
     module_build = rida.database.ModuleBuild.from_repo_done_event(session, msg)
@@ -43,23 +48,63 @@ def done(config, session, msg):
         log.info("No module build found associated with koji tag %r" % tag)
         return
 
-    unbuilt_components = (
-        component_build for component_build in module_build.component_builds
-        if component_build.state is None
-    )
+    # It is possible that we have already failed.. but our repo is just being
+    # routinely regenerated.  Just ignore that.  If rida says the module is
+    # dead, then the module is dead.
+    if module_build.state == rida.BUILD_STATES['failed']:
+        log.info("Ignoring repo regen for already failed %r" % module_build)
+        return
+
+    current_batch = module_build.current_batch()
+
+    # If any in the current batch are still running.. just wait.
+    running = [c.state == koji.BUILD_STATES['BUILDING'] for c in current_batch]
+    if any(running):
+        log.info("Module build %r has %r of %r components still building" % (
+            module_build, len(running), len(current_batch)))
+        return
+
+    # Assemble the list of all successful components in the batch.
+    good = [c for c in current_batch if c.state == koji.BUILD_STATES['COMPLETE']]
+
+    # If *none* of the components completed for this batch, then obviously the
+    # module fails.  However!  We shouldn't reach this scenario.  There is
+    # logic over in the component handler which should fail the module build
+    # first before we ever get here.  This is here as a race condition safety
+    # valve.
+    if not good:
+        module_build.transition(config, rida.BUILD_STATES['failed'])
+        session.commit()
+        log.warn("Odd!  All component builds failed for %r." % module_build)
+        return
 
     builder = rida.builder.KojiModuleBuilder(module_build.name, config, tag_name=tag)
     builder.buildroot_resume()
 
-    for component_build in unbuilt_components:
-        component_build.state = koji.BUILD_STATES['BUILDING']
-        log.debug("Using scmurl=%s for package=%s" % (
-            component_build.scmurl,
-            component_build.package,
-        ))
-        log.info("Building artifact_name=%s from source=%s" % (component_build.package, component_build.scmurl))
-        component_build.task_id = builder.build(
-            artifact_name=component_build.package,
-            source=component_build.scmurl,
-        )
-    session.commit()
+    # Ok, for the subset of builds that did complete successfully, check to
+    # see if they are in the buildroot.
+    artifacts = [component_build.package for component_build in good]
+    if not builder.buildroot_ready(artifacts):
+        log.info("Not all of %r are in the buildroot.  Waiting." % artifacts)
+        return
+
+    # If we have reached here then we know the following things:
+    #
+    # - All components in this batch have finished (failed or succeeded)
+    # - One or more succeeded.
+    # - They have been regenerated back into the buildroot.
+    #
+    # So now we can either start a new batch if there are still some to build
+    # or, if everything is built successfully, then we can bless the module as
+    # complete.
+    leftover_components = [
+        c for c in module_build.components_builds
+        if c.state != koji.BUILD_STATES['COMPLETE']
+    ]
+    if leftover_components:
+        rida.utils.start_next_build_batch(module_build, session, builder, components=leftover_components)
+    else:
+        module_build.transition(config, state=rida.BUILD_STATES['complete'])
+        session.commit()
+
+    # And that's it.  :)

rida/utils.py

@@ -24,6 +24,8 @@
 import functools
 import time
 
+import koji
+
 
 def retry(timeout=120, interval=30, wait_on=Exception):
     """ A decorator that allows to retry a section of code...
@@ -42,3 +44,25 @@ def retry(timeout=120, interval=30, wait_on=Exception):
                     time.sleep(interval)
         return inner
     return wrapper
+
+
+def start_next_build_batch(module, session, builder, components=None):
+    """ Starts a next round of the build cycle for a module. """
+
+    if any([c.state == koji.BUILD_STATES['BUILDING']
+            for c in module.component_builds ]):
+        raise ValueError("Cannot start a batch when another is in flight.")
+
+    # The user can either pass in a list of components to 'seed' the batch, or
+    # if none are provided then we just select everything that hasn't
+    # successfully built yet.
+    unbuilt_components = components or [
+        c for c in module.component_builds
+        if c.state != koji.BUILD_STATES['COMPLETE']
+    ]
+    module.batch += 1
+    for c in unbuilt_components:
+        c.batch = module.batch
+        c.task_id = builder.build(artifact_name=c.package, source=c.scmurl)
+
+    session.commit()