Allow building modules locally without fedmsg-hub.

We started using `events.scheduler` to plan fake events instead of internal
queue implemented by fedmsg-hub. Thanks to that, we can stop using fedmsg-hub
completely for local builds.

In this commit, the `scheduler.local.main` is rewritten to not use fedmsg-hub.
The original `scheduler.local.main` is still needed by test_build tests and
therefore it is moved there.

module_build_service/manage.py

@@ -182,10 +182,7 @@ def build_module_locally(
 
         module_build_ids = [build.id for build in module_builds]
 
-    stop = module_build_service.scheduler.local.make_simple_stop_condition()
-
-    # Run the consumer until stop_condition returns True
-    module_build_service.scheduler.local.main([], stop)
+    module_build_service.scheduler.local.main(module_build_ids)
 
     has_failed_module = db_session.query(models.ModuleBuild).filter(
         models.ModuleBuild.id.in_(module_build_ids),

module_build_service/scheduler/local.py

@@ -1,12 +1,14 @@
 # -*- coding: utf-8 -*-
 # SPDX-License-Identifier: MIT
 
-import fedmsg
 import logging
-import module_build_service.models
-import moksha.hub
+from module_build_service import models
 
 from module_build_service.db_session import db_session
+from module_build_service.scheduler.handlers.modules import init as modules_init_handler
+from module_build_service.scheduler.handlers.modules import wait as modules_wait_handler
+from module_build_service.scheduler.handlers.modules import done as modules_done_handler
+from module_build_service.scheduler.handlers.modules import failed as modules_failed_handler
 
 log = logging.getLogger(__name__)
 
@@ -16,82 +18,48 @@ This module contains functions to control fedmsg-hub running locally for local
 module build and running tests within test_build.py particularly.
 """
 
-__all__ = ["main", "make_simple_stop_condition"]
+__all__ = ["main"]
 
 
-def main(initial_messages, stop_condition):
-    """ Run the consumer until some condition is met.
-
-    Setting stop_condition to None will run the consumer forever.
+def raise_for_failed_build(module_build_ids):
     """
+    Raises an exception if any module build from `module_build_ids` list is in failed state.
+    This function also calls "failed" handler before raises an exception.
 
-    config = fedmsg.config.load_config()
-    config["mbsconsumer"] = True
-    config["mbsconsumer.stop_condition"] = stop_condition
-    config["mbsconsumer.initial_messages"] = initial_messages
-
-    # Moksha requires that we subscribe to *something*, so tell it /dev/null
-    # since we'll just be doing in-memory queue-based messaging for this single
-    # build.
-    config["zmq_enabled"] = True
-    config["zmq_subscribe_endpoints"] = "ipc:///dev/null"
-
-    # Lazy import consumer to avoid potential import cycle.
-    # For example, in some cases, importing event message from events.py would
-    # cause importing the consumer module, which then starts to import relative
-    # code inside handlers module, and the original code is imported eventually.
-    import module_build_service.scheduler.consumer
-
-    consumers = [module_build_service.scheduler.consumer.MBSConsumer]
-
-    # Note that the hub we kick off here cannot send any message.  You
-    # should use fedmsg.publish(...) still for that.
-    moksha.hub.main(
-        # Pass in our config dict
-        options=config,
-        # Only run the specified consumers if any are so specified.
-        consumers=consumers,
-        # Do not run default producers.
-        producers=[],
-        # Tell moksha to quiet its logging.
-        framework=False,
-    )
-
-
-def make_simple_stop_condition():
-    """ Return a simple stop_condition callable.
-
-    Intended to be used with the main() function here in manage.py and tests.
-
-    The stop_condition returns true when the latest module build enters the any
-    of the finished states.
+    :param list module_build_ids: List of module build IDs (int) to build locally.
     """
+    builds = db_session.query(models.ModuleBuild).filter(
+        models.ModuleBuild.id.in_(module_build_ids)).all()
+    has_failed_build = False
+    for build in builds:
+        if build.state == models.BUILD_STATES["failed"]:
+            modules_failed_handler("fake_msg_id", build.id, "failed")
+            has_failed_build = True
+    if has_failed_build:
+        raise ValueError("Local module build failed.")
 
-    def stop_condition(message):
-        # XXX - We ignore the message here and instead just query the DB.
 
-        # Grab the latest module build.
-        module = (
-            db_session.query(module_build_service.models.ModuleBuild)
-            .order_by(module_build_service.models.ModuleBuild.id.desc())
-            .first()
-        )
-        done = (
-            module_build_service.models.BUILD_STATES["failed"],
-            module_build_service.models.BUILD_STATES["ready"],
-            module_build_service.models.BUILD_STATES["done"],
-        )
-        result = module.state in done
-        log.debug("stop_condition checking %r, got %r" % (module, result))
+def main(module_build_ids):
+    """
+    Build modules locally. The modules have to be stored in the local database before
+    calling this function.
 
-        # moksha.hub.main starts the hub and runs it in a separate thread. When
-        # the result is True, remove the db_session from that thread local so
-        # that any pending queries in the transaction will not block other
-        # queries made from other threads.
-        # This is useful for testing particularly.
-        if result:
-            db_session.remove()
+    :param list module_build_ids: List of module build IDs (int) to build locally.
+    """
+    # The transition between states is normally handled by ModuleBuild.transition, which sends
+    # a message to message bus. The message is then received by the Consumer and handler is called.
+    # But for local builds, we do not have any message bus, so we just call the handlers in
+    # the right order manually.
+    # We only need to ensure that we won't call futher handlers in case the module build failed.
+    for module_build_id in module_build_ids:
+        modules_init_handler("fake_msg_id", module_build_id, "init")
 
-        return result
+    raise_for_failed_build(module_build_ids)
+    for module_build_id in module_build_ids:
+        modules_wait_handler("fake_msg_id", module_build_id, "wait")
 
-    return stop_condition
+    raise_for_failed_build(module_build_ids)
+    for module_build_id in module_build_ids:
+        modules_done_handler("fake_msg_id", module_build_id, "done")
+
+    raise_for_failed_build(module_build_ids)

tests/test_build/test_build.py

@@ -10,16 +10,17 @@ from shutil import copyfile
 from datetime import datetime, timedelta
 from random import randint
 import hashlib
+import moksha.hub
+import fedmsg
 
 import module_build_service.messaging
 import module_build_service.scheduler.consumer
 import module_build_service.scheduler.handlers.repos
 import module_build_service.utils
 from module_build_service.errors import Forbidden
-from module_build_service import models, conf, build_logs
+from module_build_service import models, conf, build_logs, log
 from module_build_service.db_session import db_session
 from module_build_service.scheduler import events
-from module_build_service.scheduler.local import make_simple_stop_condition
 from module_build_service.scheduler.handlers.tags import tagged as tagged_handler
 from module_build_service.scheduler.handlers.components import (
     build_task_finalize as build_task_finalize_handler)
@@ -44,6 +45,83 @@ base_dir = dirname(dirname(__file__))
 user = ("Homer J. Simpson", {"packager"})
 
 
+def make_simple_stop_condition():
+    """ Return a simple stop_condition callable.
+
+    Intended to be used with the main() function here in manage.py and tests.
+
+    The stop_condition returns true when the latest module build enters the any
+    of the finished states.
+    """
+    def stop_condition(message):
+        # XXX - We ignore the message here and instead just query the DB.
+
+        # Grab the latest module build.
+        module = (
+            db_session.query(models.ModuleBuild)
+            .order_by(models.ModuleBuild.id.desc())
+            .first()
+        )
+        done = (
+            models.BUILD_STATES["failed"],
+            models.BUILD_STATES["ready"],
+            models.BUILD_STATES["done"],
+        )
+        result = module.state in done
+        log.debug("stop_condition checking %r, got %r" % (module, result))
+
+        # moksha.hub.main starts the hub and runs it in a separate thread. When
+        # the result is True, remove the db_session from that thread local so
+        # that any pending queries in the transaction will not block other
+        # queries made from other threads.
+        # This is useful for testing particularly.
+        if result:
+            db_session.remove()
+
+        return result
+
+    return stop_condition
+
+
+def main(initial_messages, stop_condition):
+    """ Run the consumer until some condition is met.
+
+    Setting stop_condition to None will run the consumer forever.
+    """
+
+    config = fedmsg.config.load_config()
+    config["mbsconsumer"] = True
+    config["mbsconsumer.stop_condition"] = stop_condition
+    config["mbsconsumer.initial_messages"] = initial_messages
+
+    # Moksha requires that we subscribe to *something*, so tell it /dev/null
+    # since we'll just be doing in-memory queue-based messaging for this single
+    # build.
+    config["zmq_enabled"] = True
+    config["zmq_subscribe_endpoints"] = "ipc:///dev/null"
+
+    # Lazy import consumer to avoid potential import cycle.
+    # For example, in some cases, importing event message from events.py would
+    # cause importing the consumer module, which then starts to import relative
+    # code inside handlers module, and the original code is imported eventually.
+    import module_build_service.scheduler.consumer
+
+    consumers = [module_build_service.scheduler.consumer.MBSConsumer]
+
+    # Note that the hub we kick off here cannot send any message.  You
+    # should use fedmsg.publish(...) still for that.
+    moksha.hub.main(
+        # Pass in our config dict
+        options=config,
+        # Only run the specified consumers if any are so specified.
+        consumers=consumers,
+        # Do not run default producers.
+        producers=[],
+        # Tell moksha to quiet its logging.
+        framework=False,
+    )
+
+
 class FakeSCM(object):
     def __init__(self, mocked_scm, name, mmd_filename, commit=None, version=20180205135154):
         self.mocked_scm = mocked_scm
@@ -320,7 +398,7 @@ def cleanup_moksha():
 class BaseTestBuild:
 
     def run_scheduler(self, msgs=None, stop_condition=None):
-        module_build_service.scheduler.local.main(
+        main(
             msgs or [],
             stop_condition or make_simple_stop_condition()
         )

tests/test_manage.py

@@ -201,7 +201,7 @@ class TestCommandBuildModuleLocally:
             "--file", staged_data_filename("testmodule-local-build.yaml")
         ]
 
-        def main_side_effect(initial_messages, stop_condition):
+        def main_side_effect(module_build_ids):
             build = db_session.query(models.ModuleBuild).filter(
                 models.ModuleBuild.name == "testmodule-local-build"
             ).first()