Merge #1146 Update the ModuleBuild.time_modified in case the format_mmd takes long time.

module_build_service/utils/submit.py

@@ -119,13 +119,16 @@ def _scm_get_latest(pkg):
     }
 
 
-def format_mmd(mmd, scmurl):
+def format_mmd(mmd, scmurl, module=None, session=None):
     """
     Prepares the modulemd for the MBS. This does things such as replacing the
     branches of components with commit hashes and adding metadata in the xmd
     dictionary.
     :param mmd: the Modulemd.Module object to format
     :param scmurl: the url to the modulemd
+    :param module: When specified together with `session`, the time_modified
+        of a module is updated regularly in case this method takes lot of time.
+    :param session: Database session to update the `module`.
     """
     # Import it here, because SCM uses utils methods and fails to import
     # them because of dep-chain.
@@ -203,7 +206,17 @@ def format_mmd(mmd, scmurl):
             # previous runs of this method (can be caused by module build resubmition).
             pkgs_to_resolve = [pkg for pkg in mmd.get_rpm_components().values()
                                if pkg.get_name() not in xmd['mbs']['rpms']]
-            pkg_dicts = pool.map(_scm_get_latest, pkgs_to_resolve)
+            async_result = pool.map_async(_scm_get_latest, pkgs_to_resolve)
+
+            # For modules with lot of components, the _scm_get_latest can take a lot of time.
+            # We need to bump time_modified from time to time, otherwise poller could think
+            # that module is stuck in "init" state and it would send fake "init" message.
+            while not async_result.ready():
+                async_result.wait(60)
+                if module and session:
+                    module.time_modified = datetime.utcnow()
+                    session.commit()
+            pkg_dicts = async_result.get()
         finally:
             pool.close()
 
@@ -310,7 +323,7 @@ def record_component_builds(mmd, module, initial_batch=1,
 
     # Format the modulemd by putting in defaults and replacing streams that
     # are branches with commit hashes
-    format_mmd(mmd, module.scmurl)
+    format_mmd(mmd, module.scmurl, module, session)
 
     # When main_mmd is set, merge the metadata from this mmd to main_mmd,
     # otherwise our current mmd is main_mmd.

tests/test_utils/test_utils.py

@@ -630,6 +630,31 @@ class TestUtils:
             for pkg in mmd2.get_rpm_components().values():
                 assert set(pkg.get_arches().get()) == set(test_archs)
 
+    @patch('module_build_service.scm.SCM')
+    @patch('module_build_service.utils.submit.ThreadPool')
+    def test_format_mmd_update_time_modified(self, tp, mocked_scm):
+        with app.app_context():
+            init_data()
+            build = models.ModuleBuild.query.get(2)
+
+            async_result = mock.MagicMock()
+            async_result.ready.side_effect = [False, False, False, True]
+            tp.return_value.map_async.return_value = async_result
+
+            test_datetime = datetime(2019, 2, 14, 11, 11, 45, 42968)
+
+            testmodule_mmd_path = path.join(
+                BASE_DIR, '..', 'staged_data', 'testmodule.yaml')
+
+            mmd1 = Modulemd.Module().new_from_file(testmodule_mmd_path)
+            mmd1.upgrade()
+
+            with patch('module_build_service.utils.submit.datetime') as dt:
+                dt.utcnow.return_value = test_datetime
+                module_build_service.utils.format_mmd(mmd1, None, build, db.session)
+
+            assert build.time_modified == test_datetime
+
     def test_generate_koji_tag_in_nsvc_format(self):
         name, stream, version, context = ('testmodule', 'master', '20170816080815', '37c6c57')