diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index 0ede68a3..2aec8ff4 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -119,13 +119,16 @@ def _scm_get_latest(pkg): } -def format_mmd(mmd, scmurl): +def format_mmd(mmd, scmurl, module=None, session=None): """ Prepares the modulemd for the MBS. This does things such as replacing the branches of components with commit hashes and adding metadata in the xmd dictionary. :param mmd: the Modulemd.Module object to format :param scmurl: the url to the modulemd + :param module: When specified together with `session`, the time_modified + of a module is updated regularly in case this method takes lot of time. + :param session: Database session to update the `module`. """ # Import it here, because SCM uses utils methods and fails to import # them because of dep-chain. @@ -203,7 +206,17 @@ def format_mmd(mmd, scmurl): # previous runs of this method (can be caused by module build resubmition). pkgs_to_resolve = [pkg for pkg in mmd.get_rpm_components().values() if pkg.get_name() not in xmd['mbs']['rpms']] - pkg_dicts = pool.map(_scm_get_latest, pkgs_to_resolve) + async_result = pool.map_async(_scm_get_latest, pkgs_to_resolve) + + # For modules with lot of components, the _scm_get_latest can take a lot of time. + # We need to bump time_modified from time to time, otherwise poller could think + # that module is stuck in "init" state and it would send fake "init" message. + while not async_result.ready(): + async_result.wait(60) + if module and session: + module.time_modified = datetime.utcnow() + session.commit() + pkg_dicts = async_result.get() finally: pool.close() @@ -310,7 +323,7 @@ def record_component_builds(mmd, module, initial_batch=1, # Format the modulemd by putting in defaults and replacing streams that # are branches with commit hashes - format_mmd(mmd, module.scmurl) + format_mmd(mmd, module.scmurl, module, session) # When main_mmd is set, merge the metadata from this mmd to main_mmd, # otherwise our current mmd is main_mmd. diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index d3b8e944..213c9365 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -630,6 +630,31 @@ class TestUtils: for pkg in mmd2.get_rpm_components().values(): assert set(pkg.get_arches().get()) == set(test_archs) + @patch('module_build_service.scm.SCM') + @patch('module_build_service.utils.submit.ThreadPool') + def test_format_mmd_update_time_modified(self, tp, mocked_scm): + with app.app_context(): + init_data() + build = models.ModuleBuild.query.get(2) + + async_result = mock.MagicMock() + async_result.ready.side_effect = [False, False, False, True] + tp.return_value.map_async.return_value = async_result + + test_datetime = datetime(2019, 2, 14, 11, 11, 45, 42968) + + testmodule_mmd_path = path.join( + BASE_DIR, '..', 'staged_data', 'testmodule.yaml') + + mmd1 = Modulemd.Module().new_from_file(testmodule_mmd_path) + mmd1.upgrade() + + with patch('module_build_service.utils.submit.datetime') as dt: + dt.utcnow.return_value = test_datetime + module_build_service.utils.format_mmd(mmd1, None, build, db.session) + + assert build.time_modified == test_datetime + def test_generate_koji_tag_in_nsvc_format(self): name, stream, version, context = ('testmodule', 'master', '20170816080815', '37c6c57')