From 879e3befae01b6240a392bad9c41ad04426f11bf Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 9 Aug 2022 08:59:34 -0400 Subject: [PATCH 1/7] combine code for noting params in xmd --- module_build_service/web/submit.py | 39 ++++++++++++------------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/module_build_service/web/submit.py b/module_build_service/web/submit.py index 4866bffd..942790f4 100644 --- a/module_build_service/web/submit.py +++ b/module_build_service/web/submit.py @@ -270,37 +270,29 @@ def _apply_dep_overrides(mmd, params): ) -def _apply_rpm_component_ref_overrides(mmd, params): +def _apply_xmd_params(mmd, params): """ - If `rpm_component_ref_overrides` is given, note it in the xmd. + Note some parameters in the mbs section of the xmd. :param Modulemd.ModuleStream mmd: the modulemd to apply the overrides on :param dict params: the API parameters passed in by the user """ + xmd_updates = {} + ref_overrides = params.get("rpm_component_ref_overrides", {}) - if not ref_overrides: - # No changes needed. - return - - xmd = mmd.get_xmd() - xmd.setdefault("mbs", {})["rpm_component_ref_overrides"] = ref_overrides - mmd.set_xmd(xmd) - - -def _apply_side_tag(mmd, params): - """ - If a side tag identifier is given, note it in the xmd - - :param Modulemd.ModuleStream mmd: the modulemd to apply the overrides on - :param dict params: the API parameters passed in by the user - """ + if ref_overrides: + xmd_updates["rpm_component_ref_overrides"] = ref_overrides side_tag = params.get('side_tag') - if not side_tag: - # no changes needed - return + if side_tag: + xmd_updates["side_tag"] = side_tag + branch = params.get('side_tag') + if side_tag: + xmd_updates["branch"] = branch + if not xmd_updates: + return xmd = mmd.get_xmd() - xmd.setdefault("mbs", {})["side_tag"] = side_tag + xmd.setdefault("mbs", {}).update(xmd_updates) mmd.set_xmd(xmd) @@ -617,8 +609,7 @@ def submit_module_build(db_session, username, stream_or_packager, params, module for mmd in input_mmds: validate_mmd(mmd) _apply_dep_overrides(mmd, params) - _apply_side_tag(mmd, params) - _apply_rpm_component_ref_overrides(mmd, params) + _apply_xmd_params(mmd, params) _modify_buildtime_streams(db_session, mmd, resolve_base_module_virtual_streams) _process_support_streams(db_session, mmd, params) mmds += generate_expanded_mmds(db_session, mmd, raise_if_stream_ambigous, From d1180871de0e79c20899926b4ced690ce9e156fb Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 9 Aug 2022 09:12:39 -0400 Subject: [PATCH 2/7] fix unit test --- tests/test_web/test_submit.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/test_web/test_submit.py b/tests/test_web/test_submit.py index fb7aa57e..fae3e0f1 100644 --- a/tests/test_web/test_submit.py +++ b/tests/test_web/test_submit.py @@ -17,7 +17,7 @@ from module_build_service.common.utils import (mmd_to_str, load_mmd, from module_build_service.scheduler.db_session import db_session from module_build_service.web.submit import ( get_prefixed_version, submit_module_build, submit_module_build_from_yaml, - process_module_context_configuration, _apply_side_tag, + process_module_context_configuration, _apply_xmd_params, ) from tests import ( scheduler_init_data, @@ -317,20 +317,26 @@ data: """ return load_mmd(yaml_str) - def test_apply_side_tag(self): + def test_apply_xmd_params(self): """ - Test that the side tag option is correctly added into the xmd + Test that key params are correctly added into the xmd """ mmd = self.get_mmd() side_tag = "SIDETAG" - _apply_side_tag(mmd, {"side_tag": side_tag}) + params = { + "side_tag": "SIDETAG", + "rpm_component_ref_overrides": {"pkg": "f4836ea0"}, + "branch": "f37", + } + _apply_xmd_params(mmd, params) xmd = mmd.get_xmd() - assert xmd["mbs"]["side_tag"] == side_tag + for key in params: + assert xmd["mbs"][key] == params[key] def test_apply_side_tag_no_option(self): """ - Test that the xmd is unchanged when option not given + Test that the xmd is unchanged when no relevant options are given """ mmd = self.get_mmd() xmd_orig = mmd.get_xmd() From 79648cf188c94f3fae704082e200e03701182806 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 9 Aug 2022 09:54:04 -0400 Subject: [PATCH 3/7] use submitted branch as default ref for components --- module_build_service/scheduler/submit.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/module_build_service/scheduler/submit.py b/module_build_service/scheduler/submit.py index 21e1aa56..5461495f 100644 --- a/module_build_service/scheduler/submit.py +++ b/module_build_service/scheduler/submit.py @@ -281,6 +281,8 @@ def format_mmd(mmd, scmurl, module=None, db_session=None, srpm_overrides=None): xmd["mbs"]["commit"] = full_scm_hash + default_ref = xmd["mbs"].get("branch") or conf.default_branch + if mmd.get_rpm_component_names() or mmd.get_module_component_names(): if "rpms" not in xmd["mbs"]: xmd["mbs"]["rpms"] = {} @@ -309,7 +311,7 @@ def format_mmd(mmd, scmurl, module=None, db_session=None, srpm_overrides=None): if not pkg.get_cache(): pkg.set_cache(conf.rpms_default_cache + pkgname) if not pkg.get_ref(): - pkg.set_ref(conf.default_branch) + pkg.set_ref(default_ref) if not pkg.get_arches(): for arch in conf.arches: pkg.add_restricted_arch(arch) @@ -325,7 +327,7 @@ def format_mmd(mmd, scmurl, module=None, db_session=None, srpm_overrides=None): if not mod.get_repository(): mod.set_repository(conf.modules_default_repository + modname) if not mod.get_ref(): - mod.set_ref(conf.default_branch) + mod.set_ref(default_ref) # It is possible to override the ref of RPM component using # the rpm_component_ref_overrides. From 1236fed8b8d8926b56fdef9136954cf82d69a668 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 9 Aug 2022 11:48:52 -0400 Subject: [PATCH 4/7] bugfix --- module_build_service/web/submit.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/module_build_service/web/submit.py b/module_build_service/web/submit.py index 942790f4..9becdf76 100644 --- a/module_build_service/web/submit.py +++ b/module_build_service/web/submit.py @@ -279,15 +279,10 @@ def _apply_xmd_params(mmd, params): """ xmd_updates = {} - ref_overrides = params.get("rpm_component_ref_overrides", {}) - if ref_overrides: - xmd_updates["rpm_component_ref_overrides"] = ref_overrides - side_tag = params.get('side_tag') - if side_tag: - xmd_updates["side_tag"] = side_tag - branch = params.get('side_tag') - if side_tag: - xmd_updates["branch"] = branch + for param in ("rpm_component_ref_overrides", "side_tag", "branch"): + value = params.get(param) + if value: + xmd_updates[param] = value if not xmd_updates: return From 46f4f449a8d38ea9c952c144c28f41feadfbc76d Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 23 Aug 2022 10:49:38 -0400 Subject: [PATCH 5/7] partial work --- tests/test_web/test_submit.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_web/test_submit.py b/tests/test_web/test_submit.py index fae3e0f1..52102787 100644 --- a/tests/test_web/test_submit.py +++ b/tests/test_web/test_submit.py @@ -324,27 +324,28 @@ data: mmd = self.get_mmd() side_tag = "SIDETAG" params = { - "side_tag": "SIDETAG", - "rpm_component_ref_overrides": {"pkg": "f4836ea0"}, - "branch": "f37", - } + "side_tag": "SIDETAG", + "rpm_component_ref_overrides": {"pkg": "f4836ea0"}, + "branch": "f37", + } _apply_xmd_params(mmd, params) xmd = mmd.get_xmd() for key in params: assert xmd["mbs"][key] == params[key] - def test_apply_side_tag_no_option(self): + def test_apply_xmd_params_no_option(self): """ Test that the xmd is unchanged when no relevant options are given """ mmd = self.get_mmd() xmd_orig = mmd.get_xmd() - _apply_side_tag(mmd, {}) + _apply_xmd_params(mmd, {}) xmd = mmd.get_xmd() assert xmd == xmd_orig - assert "side_tag" not in xmd.get("mbs", {}) + for key in ("side_tag", "rpm_component_ref_overrides", "branch"): + assert key not in xmd.get("mbs", {}) @pytest.mark.usefixtures("reuse_component_init_data") From 9287297943794578433c3dcdfce7947c00c721ad Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Tue, 8 Nov 2022 19:23:33 -0500 Subject: [PATCH 6/7] fix unit test --- tests/test_web/test_submit.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_web/test_submit.py b/tests/test_web/test_submit.py index 52102787..47c36f25 100644 --- a/tests/test_web/test_submit.py +++ b/tests/test_web/test_submit.py @@ -322,12 +322,11 @@ data: Test that key params are correctly added into the xmd """ mmd = self.get_mmd() - side_tag = "SIDETAG" params = { "side_tag": "SIDETAG", "rpm_component_ref_overrides": {"pkg": "f4836ea0"}, "branch": "f37", - } + } _apply_xmd_params(mmd, params) xmd = mmd.get_xmd() @@ -467,4 +466,7 @@ class TestUtilsComponentReuse: ): submit_module_build(db_session, "foo", mmd_copy, {"branch": "private-foo"}, version) + # re-copy because submit_module_build could modify xmd + mmd_copy = mmd.copy() + mmd_copy.set_xmd({}) submit_module_build(db_session, "foo", mmd_copy, {"branch": "otherbranch"}, version) From a014f20f1556186e30549e8d18da9710cfd1c30c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Mon, 21 Nov 2022 15:17:30 -0500 Subject: [PATCH 7/7] handle default_ref recursively for module components --- module_build_service/scheduler/submit.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/module_build_service/scheduler/submit.py b/module_build_service/scheduler/submit.py index 5461495f..06f4f2aa 100644 --- a/module_build_service/scheduler/submit.py +++ b/module_build_service/scheduler/submit.py @@ -247,7 +247,7 @@ def _scm_get_latest(data): return {"pkg_name": data.rpm_component.get_name(), "pkg_ref": pkgref, "error": None} -def format_mmd(mmd, scmurl, module=None, db_session=None, srpm_overrides=None): +def format_mmd(mmd, scmurl, module=None, db_session=None, srpm_overrides=None, default_ref=None): """ Prepares the modulemd for the MBS. This does things such as replacing the branches of components with commit hashes and adding metadata in the xmd @@ -281,7 +281,8 @@ def format_mmd(mmd, scmurl, module=None, db_session=None, srpm_overrides=None): xmd["mbs"]["commit"] = full_scm_hash - default_ref = xmd["mbs"].get("branch") or conf.default_branch + if default_ref is None: + default_ref = xmd["mbs"].get("branch") or conf.default_branch if mmd.get_rpm_component_names() or mmd.get_module_component_names(): if "rpms" not in xmd["mbs"]: @@ -510,6 +511,8 @@ def record_component_builds( # planned for batch 2 and following. batch = initial_batch + default_ref = mmd.get_xmd()["mbs"].get("branch") + for component in all_components: # Increment the batch number when buildorder increases. if previous_buildorder != component.get_buildorder(): @@ -525,7 +528,7 @@ def record_component_builds( # It is OK to whitelist all URLs here, because the validity # of every URL have been already checked in format_mmd(...). included_mmd = fetch_mmd(full_url, whitelist_url=True)[0] - format_mmd(included_mmd, module.scmurl, module, db_session, srpm_overrides) + format_mmd(included_mmd, module.scmurl, module, db_session, srpm_overrides, default_ref) batch = record_component_builds( included_mmd, module, batch, previous_buildorder, main_mmd) continue