From efaaea66e0b1cc882ff4e4b2cece3a3e0d374b2e Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Thu, 14 Feb 2019 12:39:54 +0100 Subject: [PATCH] Update the ModuleBuild.time_modified in case the `format_mmd` takes long time. In case there is lot of components in a module build or there are some networking issues and we need to retry the "git clone" commands, the `format_mmd` method can take long time. If it takes more than 10 minutes, the poller can produce fake event, because it seems the module build is stuck. This is wrong, because it would lead to another unexpected init handler call. In this commit, the `format_mmd` updates the `ModuleBuild.time_modified` method regularly to prevent poller from sending the unexpected fake init message. --- module_build_service/utils/submit.py | 19 ++++++++++++++++--- tests/test_utils/test_utils.py | 25 +++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index 0ede68a3..2aec8ff4 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -119,13 +119,16 @@ def _scm_get_latest(pkg): } -def format_mmd(mmd, scmurl): +def format_mmd(mmd, scmurl, module=None, session=None): """ Prepares the modulemd for the MBS. This does things such as replacing the branches of components with commit hashes and adding metadata in the xmd dictionary. :param mmd: the Modulemd.Module object to format :param scmurl: the url to the modulemd + :param module: When specified together with `session`, the time_modified + of a module is updated regularly in case this method takes lot of time. + :param session: Database session to update the `module`. """ # Import it here, because SCM uses utils methods and fails to import # them because of dep-chain. @@ -203,7 +206,17 @@ def format_mmd(mmd, scmurl): # previous runs of this method (can be caused by module build resubmition). pkgs_to_resolve = [pkg for pkg in mmd.get_rpm_components().values() if pkg.get_name() not in xmd['mbs']['rpms']] - pkg_dicts = pool.map(_scm_get_latest, pkgs_to_resolve) + async_result = pool.map_async(_scm_get_latest, pkgs_to_resolve) + + # For modules with lot of components, the _scm_get_latest can take a lot of time. + # We need to bump time_modified from time to time, otherwise poller could think + # that module is stuck in "init" state and it would send fake "init" message. + while not async_result.ready(): + async_result.wait(60) + if module and session: + module.time_modified = datetime.utcnow() + session.commit() + pkg_dicts = async_result.get() finally: pool.close() @@ -310,7 +323,7 @@ def record_component_builds(mmd, module, initial_batch=1, # Format the modulemd by putting in defaults and replacing streams that # are branches with commit hashes - format_mmd(mmd, module.scmurl) + format_mmd(mmd, module.scmurl, module, session) # When main_mmd is set, merge the metadata from this mmd to main_mmd, # otherwise our current mmd is main_mmd. diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index d3b8e944..213c9365 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -630,6 +630,31 @@ class TestUtils: for pkg in mmd2.get_rpm_components().values(): assert set(pkg.get_arches().get()) == set(test_archs) + @patch('module_build_service.scm.SCM') + @patch('module_build_service.utils.submit.ThreadPool') + def test_format_mmd_update_time_modified(self, tp, mocked_scm): + with app.app_context(): + init_data() + build = models.ModuleBuild.query.get(2) + + async_result = mock.MagicMock() + async_result.ready.side_effect = [False, False, False, True] + tp.return_value.map_async.return_value = async_result + + test_datetime = datetime(2019, 2, 14, 11, 11, 45, 42968) + + testmodule_mmd_path = path.join( + BASE_DIR, '..', 'staged_data', 'testmodule.yaml') + + mmd1 = Modulemd.Module().new_from_file(testmodule_mmd_path) + mmd1.upgrade() + + with patch('module_build_service.utils.submit.datetime') as dt: + dt.utcnow.return_value = test_datetime + module_build_service.utils.format_mmd(mmd1, None, build, db.session) + + assert build.time_modified == test_datetime + def test_generate_koji_tag_in_nsvc_format(self): name, stream, version, context = ('testmodule', 'master', '20170816080815', '37c6c57')