From 28d684331f8449033bb14a9e6e0285253704e5df Mon Sep 17 00:00:00 2001
From: Matt Prahl <mprahl@redhat.com>
Date: Wed, 16 Nov 2016 15:23:51 -0500
Subject: [PATCH 1/7] Remove stale code from previous modulemd version

---
 .../scheduler/handlers/components.py           | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/module_build_service/scheduler/handlers/components.py b/module_build_service/scheduler/handlers/components.py
index ec721a91..3b4f1948 100644
--- a/module_build_service/scheduler/handlers/components.py
+++ b/module_build_service/scheduler/handlers/components.py
@@ -87,24 +87,6 @@ def _finalize(config, session, msg, state):
         builder.buildroot_add_artifacts([nvr,], install=install)
         session.commit()
 
-    # Find all of the sibling builds of this particular build.
-    current_batch = parent.current_batch()
-
-    # If there are no components, transition module build state to 'done'
-    if not current_batch:
-        log.info("Batch is empty. Moving module state to DONE.")
-        parent.transition(config, module.BUILD_STATES['done'])
-        session.commit()
-        return
-
-    # Otherwise, check to see if all failed.  If so, then we cannot continue on
-    # to a next batch.  This module build is doomed.
-    if all([c.state != koji.BUILD_STATES['COMPLETE'] for c in current_batch]):
-        # They didn't all succeed.. so mark this module build as a failure.
-        parent.transition(config, models.BUILD_STATES['failed'], state_reason)
-        session.commit()
-        return
-
 
 def complete(config, session, msg):
     return _finalize(config, session, msg, state=koji.BUILD_STATES['COMPLETE'])

From 62654f871fed7d595874b5c61a175699f275af59 Mon Sep 17 00:00:00 2001
From: Matt Prahl <mprahl@redhat.com>
Date: Wed, 16 Nov 2016 09:24:08 -0500
Subject: [PATCH 2/7] Fixes bug where the build tag was not returned

---
 module_build_service/scheduler/handlers/modules.py | 5 ++++-
 tests/test_scheduler/test_module_wait.py           | 1 +
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py
index 2875dd51..c4e1499d 100644
--- a/module_build_service/scheduler/handlers/modules.py
+++ b/module_build_service/scheduler/handlers/modules.py
@@ -160,4 +160,7 @@ def wait(config, session, msg):
     # If this build already exists and is done, then fake the repo change event
     # back to the scheduler
     if state == koji.BUILD_STATES['COMPLETE']:
-        return [module_build_service.messaging.KojiRepoChange('fake msg', build.koji_tag)]
+        # TODO: builder.module_build_tag only works for Koji, figure out if
+        # other backends need this implemented (e.g. COPR)
+        return [module_build_service.messaging.KojiRepoChange(
+            'fake msg', builder.module_build_tag['name'])]
diff --git a/tests/test_scheduler/test_module_wait.py b/tests/test_scheduler/test_module_wait.py
index da3f0bd1..c07b1f2f 100644
--- a/tests/test_scheduler/test_module_wait.py
+++ b/tests/test_scheduler/test_module_wait.py
@@ -40,6 +40,7 @@ class TestModuleWait(unittest.TestCase):
         builder = mock.Mock()
         builder.get_disttag_srpm.return_value = 'some srpm disttag'
         builder.build.return_value = 1234, 1, "", None
+        builder.module_build_tag = {'name': 'some-tag-build'}
         KojiModuleBuilder.return_value = builder
         mocked_module_build = mock.Mock()
         mocked_module_build.json.return_value = {

From 02c245961bda60c37a26eb5df87958765cbdeddf Mon Sep 17 00:00:00 2001
From: Matt Prahl <mprahl@redhat.com>
Date: Tue, 15 Nov 2016 10:31:03 -0500
Subject: [PATCH 3/7] Start fedmsg-relay on boot in Vagrant

---
 Vagrantfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Vagrantfile b/Vagrantfile
index 2818042a..caa6c9e8 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -27,6 +27,7 @@ $script = <<SCRIPT
     ln -s /opt/module_build_service/krb5-stg.fp.o /etc/krb5.conf.d/stg_fedoraproject_org
     python manage.py upgradedb
     python manage.py generatelocalhostcert
+    systemctl enable fedmsg-relay
     systemctl start fedmsg-relay
     echo "export KRB5CCNAME=FILE:/var/tmp/krbcc" >> ~/.bashrc
 SCRIPT

From fb043cd2c733a384e71cb7c93d06936b724f274f Mon Sep 17 00:00:00 2001
From: Matt Prahl <mprahl@redhat.com>
Date: Thu, 10 Nov 2016 14:13:38 -0500
Subject: [PATCH 4/7] Add TODO comment

---
 module_build_service/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/module_build_service/utils.py b/module_build_service/utils.py
index 42be3390..40beebbc 100644
--- a/module_build_service/utils.py
+++ b/module_build_service/utils.py
@@ -291,6 +291,8 @@ def submit_module_build(username, url):
         err_msgs = pool.map(lambda data: "Cannot checkout {}".format(data[0])
                             if not module_build_service.scm.SCM(data[1]).is_available()
                             else None, full_urls)
+        # TODO: only the first error message is raised, perhaps concatenate
+        # the messages together?
         for err_msg in err_msgs:
             if err_msg:
                 raise UnprocessableEntity(err_msg)

From 2cb7ce99c6648ae1134ec601e087abbcde3b40c1 Mon Sep 17 00:00:00 2001
From: Matt Prahl <mprahl@redhat.com>
Date: Thu, 10 Nov 2016 13:36:47 -0500
Subject: [PATCH 5/7] Add NUM_CONSECUTIVE_BUILDS configuration option

---
 config.py                      |  4 ++++
 module_build_service/config.py | 11 +++++++++++
 2 files changed, 15 insertions(+)

diff --git a/config.py b/config.py
index d5e27caf..04eb4a0e 100644
--- a/config.py
+++ b/config.py
@@ -28,6 +28,10 @@ class BaseConfiguration(object):
     # Set to zero to disable polling
     POLLING_INTERVAL = 600
 
+    # Determines how many builds that can be submitted to the builder
+    # and be in the build state at a time. Set this to 0 for no restrictions
+    NUM_CONSECUTIVE_BUILDS = 5
+
     RPMS_DEFAULT_REPOSITORY = 'git://pkgs.fedoraproject.org/rpms/'
     RPMS_ALLOW_REPOSITORY = False
     RPMS_DEFAULT_CACHE = 'http://pkgs.fedoraproject.org/repo/pkgs/'
diff --git a/module_build_service/config.py b/module_build_service/config.py
index f6937dbf..c03cd949 100644
--- a/module_build_service/config.py
+++ b/module_build_service/config.py
@@ -200,6 +200,10 @@ class Config(object):
             'type': list,
             'default': [],
             'desc': 'Allowed SCM URLs.'},
+        'num_consecutive_builds': {
+            'type': int,
+            'default': 0,
+            'desc': 'Number of consecutive component builds.'},
     }
 
     def __init__(self):
@@ -301,3 +305,10 @@ class Config(object):
         if not isinstance(l, list):
             raise TypeError("scmurls needs to be a list.")
         self.scmurls = [str(x) for x in l]
+
+    def _setifok_num_consecutive_builds(self, i):
+        if not isinstance(i, int):
+            raise TypeError('NUM_CONSECUTIVE_BUILDS needs to be an int')
+        if i < 0:
+            raise ValueError('NUM_CONSECUTIVE_BUILDS must be >= 0')
+        self.num_consecutive_builds = i

From 67d07023c5d4c87fc7e51177a0b0ff997e226bb7 Mon Sep 17 00:00:00 2001
From: Matt Prahl <mprahl@redhat.com>
Date: Sat, 12 Nov 2016 21:00:03 -0500
Subject: [PATCH 6/7] Rename function to start_build_batch for further
 flexibility in the future

---
 module_build_service/scheduler/handlers/repos.py | 3 ++-
 module_build_service/utils.py                    | 5 +++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/module_build_service/scheduler/handlers/repos.py b/module_build_service/scheduler/handlers/repos.py
index c6c3ff1e..9f97a9fa 100644
--- a/module_build_service/scheduler/handlers/repos.py
+++ b/module_build_service/scheduler/handlers/repos.py
@@ -107,7 +107,8 @@ def done(config, session, msg):
     ]
 
     if leftover_components:
-        module_build_service.utils.start_next_build_batch(
+        module_build.batch += 1
+        module_build_service.utils.start_build_batch(
             config, module_build, session, builder)
     else:
         module_build.transition(config, state=models.BUILD_STATES['done'])
diff --git a/module_build_service/utils.py b/module_build_service/utils.py
index 40beebbc..12790563 100644
--- a/module_build_service/utils.py
+++ b/module_build_service/utils.py
@@ -30,6 +30,7 @@ import shutil
 import tempfile
 import os
 import modulemd
+import time
 from module_build_service import log, models
 from module_build_service.errors import ValidationError, UnprocessableEntity
 from module_build_service import app, conf, db, log
@@ -58,8 +59,8 @@ def retry(timeout=120, interval=30, wait_on=Exception):
     return wrapper
 
 
-def start_next_build_batch(config, module, session, builder, components=None):
-    """ Starts a next round of the build cycle for a module. """
+def start_build_batch(config, module, session, builder, components=None):
+    """ Starts a round of the build cycle for a module. """
 
     import koji  # Placed here to avoid py2/py3 conflicts...
 

From 244b77f54cdcd68c9b8bc25573d07e028d8acc0f Mon Sep 17 00:00:00 2001
From: Matt Prahl <mprahl@redhat.com>
Date: Sat, 12 Nov 2016 21:01:45 -0500
Subject: [PATCH 7/7] Add component build throttling

---
 module_build_service/models.py                | 16 ++++++---
 .../scheduler/handlers/repos.py               | 16 ++++++---
 module_build_service/scheduler/main.py        | 20 +++++++++--
 module_build_service/utils.py                 | 33 ++++++++++++++++---
 tests/test_scheduler/test_repo_done.py        |  8 +++--
 5 files changed, 73 insertions(+), 20 deletions(-)

diff --git a/module_build_service/models.py b/module_build_service/models.py
index 65ddd0ab..14cf4f63 100644
--- a/module_build_service/models.py
+++ b/module_build_service/models.py
@@ -122,16 +122,22 @@ class ModuleBuild(RidaBase):
 
     module = db.relationship('Module', backref='module_builds', lazy=False)
 
-    def current_batch(self):
+    def current_batch(self, state=None):
         """ Returns all components of this module in the current batch. """
 
         if not self.batch:
             raise ValueError("No batch is in progress: %r" % self.batch)
 
-        return [
-            component for component in self.component_builds
-            if component.batch == self.batch
-        ]
+        if state:
+            return [
+                component for component in self.component_builds
+                if component.batch == self.batch and component.state == state
+            ]
+        else:
+            return [
+                component for component in self.component_builds
+                if component.batch == self.batch
+            ]
 
     def mmd(self):
         mmd = _modulemd.ModuleMetadata()
diff --git a/module_build_service/scheduler/handlers/repos.py b/module_build_service/scheduler/handlers/repos.py
index 9f97a9fa..4ead3a7b 100644
--- a/module_build_service/scheduler/handlers/repos.py
+++ b/module_build_service/scheduler/handlers/repos.py
@@ -100,18 +100,24 @@ def done(config, session, msg):
     # So now we can either start a new batch if there are still some to build
     # or, if everything is built successfully, then we can bless the module as
     # complete.
-    leftover_components = [
+    unbuilt_components = [
         c for c in module_build.component_builds
         if (c.state != koji.BUILD_STATES['COMPLETE']
             and c.state != koji.BUILD_STATES["FAILED"])
     ]
 
-    if leftover_components:
-        module_build.batch += 1
+    if unbuilt_components:
+        # Increment the build batch when no components are being built and all
+        # have at least attempted a build (even failures) in the current batch
+        unbuilt_components_in_batch = [
+            c for c in module_build.current_batch()
+            if c.state == koji.BUILD_STATES['BUILDING'] or not c.state
+        ]
+        if not unbuilt_components_in_batch:
+            module_build.batch += 1
+
         module_build_service.utils.start_build_batch(
             config, module_build, session, builder)
     else:
         module_build.transition(config, state=models.BUILD_STATES['done'])
         session.commit()
-
-    # And that's it.  :)
diff --git a/module_build_service/scheduler/main.py b/module_build_service/scheduler/main.py
index 22247365..d9c54b3c 100644
--- a/module_build_service/scheduler/main.py
+++ b/module_build_service/scheduler/main.py
@@ -38,6 +38,7 @@ import six.moves.queue as queue
 
 import module_build_service.config
 import module_build_service.messaging
+import module_build_service.utils
 import module_build_service.scheduler.handlers.components
 import module_build_service.scheduler.handlers.modules
 import module_build_service.scheduler.handlers.repos
@@ -190,8 +191,8 @@ class Poller(threading.Thread):
                 # XXX: detect whether it's really stucked first
                 # self.process_waiting_module_builds(session)
                 self.process_open_component_builds(session)
-                self.process_lingering_module_builds(session)
                 self.fail_lost_builds(session)
+                self.process_paused_module_builds(conf, session)
 
             log.info("Polling thread sleeping, %rs" % conf.polling_interval)
             time.sleep(conf.polling_interval)
@@ -281,8 +282,21 @@ class Poller(threading.Thread):
     def process_open_component_builds(self, session):
         log.warning("process_open_component_builds is not yet implemented...")
 
-    def process_lingering_module_builds(self, session):
-        log.warning("process_lingering_module_builds is not yet implemented...")
+    def process_paused_module_builds(self, config, session):
+        if module_build_service.utils.at_concurrent_component_threshold(
+                config, session):
+            log.debug('Will not attempt to start paused module builds due to '
+                      'the concurrent build threshold being met')
+            return
+        # Check to see if module builds that are in build state but don't have
+        # any component builds being built can be worked on
+        for module_build in session.query(models.ModuleBuild).filter_by(
+                    state=models.BUILD_STATES['build']).all():
+            # If there are no components in the build state on the module build,
+            # then no possible event will start off new component builds
+            if not module_build.current_batch(koji.BUILD_STATES['BUILDING']):
+                module_build_service.utils.start_build_batch(
+                    config, module_build, session, config.system)
 
 
 _work_queue = queue.Queue()
diff --git a/module_build_service/utils.py b/module_build_service/utils.py
index 12790563..e2e8a342 100644
--- a/module_build_service/utils.py
+++ b/module_build_service/utils.py
@@ -30,7 +30,6 @@ import shutil
 import tempfile
 import os
 import modulemd
-import time
 from module_build_service import log, models
 from module_build_service.errors import ValidationError, UnprocessableEntity
 from module_build_service import app, conf, db, log
@@ -59,9 +58,28 @@ def retry(timeout=120, interval=30, wait_on=Exception):
     return wrapper
 
 
+def at_concurrent_component_threshold(config, session):
+    """
+    Determines if the number of concurrent component builds has reached
+    the configured threshold
+    :param config: Module Build Service configuration object
+    :param session: SQLAlchemy database session
+    :return: boolean representing if there are too many concurrent builds at
+    this time
+    """
+
+    import koji  # Placed here to avoid py2/py3 conflicts...
+
+    if config.num_consecutive_builds and config.num_consecutive_builds <= \
+        session.query(models.ComponentBuild).filter_by(
+            state=koji.BUILD_STATES['BUILDING']).count():
+        return True
+
+    return False
+
+
 def start_build_batch(config, module, session, builder, components=None):
     """ Starts a round of the build cycle for a module. """
-
     import koji  # Placed here to avoid py2/py3 conflicts...
 
     if any([c.state == koji.BUILD_STATES['BUILDING']
@@ -70,11 +88,11 @@ def start_build_batch(config, module, session, builder, components=None):
 
     # The user can either pass in a list of components to 'seed' the batch, or
     # if none are provided then we just select everything that hasn't
-    # successfully built yet.
-    module.batch += 1
+    # successfully built yet or isn't currently being built.
     unbuilt_components = components or [
         c for c in module.component_builds
         if (c.state != koji.BUILD_STATES['COMPLETE']
+            and c.state != koji.BUILD_STATES['BUILDING']
             and c.batch == module.batch)
     ]
 
@@ -82,7 +100,12 @@ def start_build_batch(config, module, session, builder, components=None):
         unbuilt_components))
 
     for c in unbuilt_components:
-        c.task_id, c.state, c.state_reason, c.nvr = builder.build(artifact_name=c.package, source=c.scmurl)
+        if at_concurrent_component_threshold(config, session):
+            log.info('Concurrent build threshold met')
+            break
+
+        c.task_id, c.state, c.state_reason, c.nvr = builder.build(
+            artifact_name=c.package, source=c.scmurl)
 
         if not c.task_id:
             module.transition(config, models.BUILD_STATES["failed"],
diff --git a/tests/test_scheduler/test_repo_done.py b/tests/test_scheduler/test_repo_done.py
index 603d8818..0db7a769 100644
--- a/tests/test_scheduler/test_repo_done.py
+++ b/tests/test_scheduler/test_repo_done.py
@@ -54,7 +54,9 @@ class TestRepoDone(unittest.TestCase):
     @mock.patch('module_build_service.builder.KojiModuleBuilder.build')
     @mock.patch('module_build_service.builder.KojiModuleBuilder.buildroot_connect')
     @mock.patch('module_build_service.models.ModuleBuild.from_repo_done_event')
-    def test_a_single_match(self, from_repo_done_event, connect, build_fn, config, ready):
+    @mock.patch('module_build_service.utils.at_concurrent_component_threshold',
+                return_value=False)
+    def test_a_single_match(self, threshold, from_repo_done_event, connect, build_fn, config, ready):
         """ Test that when a repo msg hits us and we have a single match.
         """
         config.return_value = mock.Mock(), "development"
@@ -88,7 +90,9 @@ class TestRepoDone(unittest.TestCase):
     @mock.patch('module_build_service.builder.KojiModuleBuilder.build')
     @mock.patch('module_build_service.builder.KojiModuleBuilder.buildroot_connect')
     @mock.patch('module_build_service.models.ModuleBuild.from_repo_done_event')
-    def test_a_single_match_build_fail(self, from_repo_done_event, connect, build_fn, config, ready):
+    @mock.patch('module_build_service.utils.at_concurrent_component_threshold',
+                return_value=False)
+    def test_a_single_match_build_fail(self, threshold, from_repo_done_event, connect, build_fn, config, ready):
         """ Test that when a KojiModuleBuilder.build fails, the build is
         marked as failed with proper state_reason.
         """