Nudge non-scratch builds stuck in done

Fixes #1783

module_build_service/common/models.py

@@ -741,15 +741,21 @@ class ModuleBuild(MBSBase):
         return local_modules
 
     @classmethod
-    def by_state(cls, db_session, state):
+    def by_state(cls, db_session, state, include_scratch=True):
         """Get module builds by state
 
         :param db_session: SQLAlchemy session object.
         :param str state: state name. Refer to key names of ``models.BUILD_STATES``.
+        :param bool include_scratch: Include scratch builds in result
         :return: a list of module builds in the specified state.
         :rtype: list[:class:`ModuleBuild`]
         """
-        return db_session.query(ModuleBuild).filter_by(state=BUILD_STATES[state]).all()
+        if include_scratch:
+            return db_session.query(ModuleBuild).filter_by(state=BUILD_STATES[state]).all()
+
+        else:
+            return db_session.query(ModuleBuild).filter_by(state=BUILD_STATES[state],
+                                                           scratch=False).all()
 
     @classmethod
     def get_by_tag(cls, db_session, tag_name):

module_build_service/scheduler/producer.py

@@ -61,7 +61,7 @@ def log_summary():
 
 @celery_app.task
 def process_waiting_module_builds():
-    for state in ["init", "wait"]:
+    for state in ["init", "wait", "done"]:
         nudge_module_builds_in_state(state, 10)
 
 
@@ -72,7 +72,12 @@ def nudge_module_builds_in_state(state_name, older_than_minutes):
     work queue.
     """
     log.info("Looking for module builds stuck in the %s state", state_name)
-    builds = models.ModuleBuild.by_state(db_session, state_name)
+    # Do not check scratch builds in done state
+    if state_name == "done":
+        include_scratch = False
+    else:
+        include_scratch = True
+    builds = models.ModuleBuild.by_state(db_session, state_name, include_scratch)
     log.info(" %r module builds in the %s state...", len(builds), state_name)
     now = datetime.utcnow()
     time_modified_threshold = timedelta(minutes=older_than_minutes)

tests/test_scheduler/test_poller.py

@@ -317,10 +317,11 @@ class TestPoller:
 
             koji_session.deleteBuildTarget.assert_not_called()
 
-    @pytest.mark.parametrize("state", ["init", "wait"])
+    @pytest.mark.parametrize("state", ["init", "wait", "done"])
     @patch.dict(producer.ON_MODULE_CHANGE_HANDLERS, clear=True, values={
         models.BUILD_STATES["init"]: mock.Mock(),
         models.BUILD_STATES["wait"]: mock.Mock(),
+        models.BUILD_STATES["done"]: mock.Mock(),
     })
     def test_process_waiting_module_build(self, create_builder, dbg, state):
         """ Test that processing old waiting module builds works. """
@@ -350,10 +351,36 @@ class TestPoller:
         # ensure the time_modified was changed.
         assert module_build.time_modified > original
 
-    @pytest.mark.parametrize("state", ["init", "wait"])
+    @pytest.mark.parametrize("state", ["done"])
+    @patch.dict(producer.ON_MODULE_CHANGE_HANDLERS, clear=True, values={
+        models.BUILD_STATES["done"]: mock.Mock(),
+    })
+    def test_process_waiting_module_build_scratch(self, create_builder, dbg, state):
+        """ Test that scratch builds in done do not get nudged. """
+
+        handler = producer.ON_MODULE_CHANGE_HANDLERS[models.BUILD_STATES[state]]
+
+        # Change the batch to 2, so the module build is in state where
+        # it is not building anything, but the state is "build".
+        module_build = models.ModuleBuild.get_by_id(db_session, 3)
+        module_build.state = models.BUILD_STATES[state]
+        original = datetime.utcnow() - timedelta(minutes=11)
+        module_build.time_modified = original
+        module_build.scratch = True
+
+        db_session.commit()
+        db_session.refresh(module_build)
+
+        # Poll :)
+        producer.process_waiting_module_builds()
+
+        handler.assert_not_called()
+
+    @pytest.mark.parametrize("state", ["init", "wait", "done"])
     @patch.dict(producer.ON_MODULE_CHANGE_HANDLERS, clear=True, values={
         models.BUILD_STATES["init"]: mock.Mock(),
         models.BUILD_STATES["wait"]: mock.Mock(),
+        models.BUILD_STATES["done"]: mock.Mock(),
     })
     def test_process_waiting_module_build_not_old_enough(
         self, create_builder, dbg, state
@@ -380,6 +407,7 @@ class TestPoller:
     @patch.dict(producer.ON_MODULE_CHANGE_HANDLERS, clear=True, values={
         models.BUILD_STATES["init"]: mock.Mock(),
         models.BUILD_STATES["wait"]: mock.Mock(),
+        models.BUILD_STATES["done"]: mock.Mock(),
     })
     def test_process_waiting_module_build_none_found(self, create_builder, dbg):
         """ Test nothing happens when no module builds are waiting. """