Merge #1673 Improve exception and SIGINT handling for tests/test_build

module_build_service/scheduler/events.py

@@ -81,6 +81,45 @@ class Scheduler(sched.scheduler):
 scheduler = Scheduler(time.time, delayfunc=lambda x: x)
 
 
+class EventTrap():
+    """
+    A context handler that can be used to provide a place to store exceptions
+    that occur in event handlers during a section of code. This is global rather
+    than per-thread, because we we set up the trap in the main thread, but
+    event event handlers are processed in the thread where moksha runs MBSConsumer.
+
+    This is needed because @celery_app.task simply logs and ignores exceptions.
+    """
+    lock = threading.Lock()
+    current = None
+
+    def __init__(self):
+        self.exception = None
+
+    def __enter__(self):
+        with EventTrap.lock:
+            EventTrap.current = self
+            return self
+
+    def __exit__(self, type, value, tb):
+        with EventTrap.lock:
+            EventTrap.current = None
+
+    @classmethod
+    def set_exception(cls, exception):
+        with cls.lock:
+            if cls.current and not cls.current.exception:
+                cls.current.exception = exception
+
+    @classmethod
+    def get_exception(cls):
+        with cls.lock:
+            if cls.current:
+                return cls.current.exception
+            else:
+                return None
+
+
 def mbs_event_handler(func):
     """
     A decorator for MBS event handlers. It implements common tasks which should otherwise
@@ -92,6 +131,9 @@ def mbs_event_handler(func):
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
+        except Exception as e:
+            EventTrap.set_exception(e)
+            raise e
         finally:
             scheduler.run()
     # save origin function as functools.wraps from python2 doesn't preserve the signature

tests/test_build/test_build.py

@@ -10,6 +10,7 @@ from os import path, mkdir
 from os.path import dirname
 import re
 import sched
+import signal
 from random import randint
 from shutil import copyfile
 
@@ -83,6 +84,16 @@ def make_simple_stop_condition():
     return stop_condition
 
 
+def make_trapped_stop_condition(stop_condition):
+    def trapped_stop_condition(message):
+        if events.EventTrap.get_exception():
+            return True
+
+        return stop_condition(message)
+
+    return trapped_stop_condition
+
+
 def main(initial_messages, stop_condition):
     """ Run the consumer until some condition is met.
 
@@ -108,18 +119,42 @@ def main(initial_messages, stop_condition):
 
     consumers = [module_build_service.scheduler.consumer.MBSConsumer]
 
-    # Note that the hub we kick off here cannot send any message.  You
-    # should use fedmsg.publish(...) still for that.
-    moksha.hub.main(
-        # Pass in our config dict
-        options=config,
-        # Only run the specified consumers if any are so specified.
-        consumers=consumers,
-        # Do not run default producers.
-        producers=[],
-        # Tell moksha to quiet its logging.
-        framework=False,
-    )
+    old_run = moksha.hub.reactor.reactor.run
+    old_sigint_handler = signal.getsignal(signal.SIGINT)
+
+    def trap_sigint(self, *args):
+        try:
+            raise KeyboardInterrupt()
+        except KeyboardInterrupt as e:
+            events.EventTrap.set_exception(e)
+
+    def set_signals_and_run(*args, **kwargs):
+        signal.signal(signal.SIGINT, trap_sigint)
+        try:
+            old_run(*args, **kwargs)
+        finally:
+            signal.signal(signal.SIGINT, old_sigint_handler)
+
+    with patch('moksha.hub.reactor.reactor.run', set_signals_and_run):
+        # The events.EventTrap context handler allows us to detect exceptions
+        # in event handlers and re-raise them here so that tests fail usefully
+        # rather than just hang.
+        with events.EventTrap() as trap:
+            # Note that the hub we kick off here cannot send any message.  You
+            # should use fedmsg.publish(...) still for that.
+            moksha.hub.main(
+                # Pass in our config dict
+                options=config,
+                # Only run the specified consumers if any are so specified.
+                consumers=consumers,
+                # Do not run default producers.
+                producers=[],
+                # Tell moksha to quiet its logging.
+                framework=False,
+            )
+
+            if trap.exception:
+                raise trap.exception
 
 
 class FakeSCM(object):
@@ -388,9 +423,10 @@ def cleanup_moksha():
 class BaseTestBuild:
 
     def run_scheduler(self, msgs=None, stop_condition=None):
+        stop_condition = stop_condition or make_simple_stop_condition()
         main(
             msgs or [],
-            stop_condition or make_simple_stop_condition()
+            make_trapped_stop_condition(stop_condition)
         )