modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-02-02 20:59:06 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

PR#1725: Allow overriding RPM components refs while submitting the module build.

Browse Source 
Merges #1725
https://pagure.io/fm-orchestrator/pull-request/1725
This commit is contained in:
Mike McLean
2021-07-28 10:07:41 -04:00
parent 31412a464f bfd9a13205
commit 14ce0c41c6
6 changed files with 133 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.rst
View File

@@ -151,6 +151,8 @@ Options:
``multipart/form-data`` request. Only allowed if ``scratch`` is ``True`` or if the MBS
setting ``YAML_SUBMIT_ALLOWED`` is ``True``. The basename of the file will be used as
the module name.
- ``rpm_component_ref_overrides`` - the commit refs to override the RPM component refs with.
The expected format is ``{'rpm_component_name': "new_commit_ref"}``.
Module build state query

48
module_build_service/scheduler/submit.py
View File

@@ -5,6 +5,7 @@ from datetime import datetime
import json
from multiprocessing.dummy import Pool as ThreadPool
import os
from collections import namedtuple
import kobo.rpmlib
@@ -17,6 +18,14 @@ from module_build_service.common.utils import to_text_type
from module_build_service.scheduler.db_session import db_session
# The namedtuple to store data passed to _scm_get_latest with following keys:
# - rpm_component - The Modulemd.ComponentRPM instance representing
# the component.
# - ref_override - The str instance used to override the component's ref
# or None.
SCMGetLatestData = namedtuple("SCMGetLatestData", "rpm_component ref_override")
def get_build_arches(mmd, config):
"""
Returns the list of architectures for which the module `mmd` should be built.
@@ -205,24 +214,35 @@ def record_filtered_rpms(mmd):
return mmd
def _scm_get_latest(pkg):
def _scm_get_latest(data):
"""
Resolves the git ref for the package defined in the `data`. Applies git
ref override if set in the `data`. Returns a dict with resolved git ref
or possible resolving error.
:param SCMGetLatestData data: Information about the packages to resolve.
:return: Dict with following keys:
- pkg_name - Name of the resolved package.
- pkg_ref - Resolve git ref.
- error - Contains the error if any or it is set to `None`.
"""
try:
# If the modulemd specifies that the 'f25' branch is what
# we want to pull from, we need to resolve that f25 branch
# to the specific commit available at the time of
# submission (now).
repo = pkg.get_repository()
ref = pkg.get_ref()
repo = data.rpm_component.get_repository()
ref = data.ref_override or data.rpm_component.get_ref()
log.debug("Getting the commit hash for the ref %s on the repo %s", ref, repo)
pkgref = module_build_service.common.scm.SCM(repo).get_latest(ref)
except Exception as e:
log.exception(e)
return {
"error": "Failed to get the latest commit for %s#%s"
% (pkg.get_repository(), pkg.get_ref())
% (data.rpm_component.get_repository(), data.rpm_component.get_ref())
}
return {"pkg_name": pkg.get_name(), "pkg_ref": pkgref, "error": None}
return {"pkg_name": data.rpm_component.get_name(), "pkg_ref": pkgref, "error": None}
def format_mmd(mmd, scmurl, module=None, db_session=None, srpm_overrides=None):
@@ -305,6 +325,10 @@ def format_mmd(mmd, scmurl, module=None, db_session=None, srpm_overrides=None):
if not mod.get_ref():
mod.set_ref("master")
# It is possible to override the ref of RPM component using
# the rpm_component_ref_overrides.
ref_overrides = xmd["mbs"].get("rpm_component_ref_overrides", {})
# Check that SCM URL is valid and replace potential branches in pkg refs
# by real SCM hash and store the result to our private xmd place in modulemd.
pool = ThreadPool(20)
@@ -312,7 +336,7 @@ def format_mmd(mmd, scmurl, module=None, db_session=None, srpm_overrides=None):
# Filter out the packages which we have already resolved in possible
# previous runs of this method (can be caused by module build resubmition)
# or which have custom SRPMs and shouldn't be resolved.
pkgs_to_resolve = []
scm_get_latest_data_list = []
for name in mmd.get_rpm_component_names():
if name not in xmd["mbs"]["rpms"]:
if name in srpm_overrides:
@@ -320,9 +344,17 @@ def format_mmd(mmd, scmurl, module=None, db_session=None, srpm_overrides=None):
# ref entry so no further verification takes place.
xmd["mbs"]["rpms"][name] = {"ref": None}
else:
pkgs_to_resolve.append(mmd.get_rpm_component(name))
# Apply possible ref override.
if name in ref_overrides:
ref_override = ref_overrides[name]
log.info("Applying rpm_component_ref_overrides - "
"%s, new ref is %s." % (name, ref_override))
else:
ref_override = None
scm_get_latest_data_list.append(
SCMGetLatestData(mmd.get_rpm_component(name), ref_override))
async_result = pool.map_async(_scm_get_latest, pkgs_to_resolve)
async_result = pool.map_async(_scm_get_latest, scm_get_latest_data_list)
# For modules with lot of components, the _scm_get_latest can take a lot of time.
# We need to bump time_modified from time to time, otherwise poller could think

18
module_build_service/web/submit.py
View File

@@ -270,6 +270,23 @@ def _apply_dep_overrides(mmd, params):
)
def _apply_rpm_component_ref_overrides(mmd, params):
"""
If `rpm_component_ref_overrides` is given, note it in the xmd.
:param Modulemd.ModuleStream mmd: the modulemd to apply the overrides on
:param dict params: the API parameters passed in by the user
"""
ref_overrides = params.get("rpm_component_ref_overrides", {})
if not ref_overrides:
# No changes needed.
return
xmd = mmd.get_xmd()
xmd.setdefault("mbs", {})["rpm_component_ref_overrides"] = ref_overrides
mmd.set_xmd(xmd)
def _apply_side_tag(mmd, params):
"""
If a side tag identifier is given, note it in the xmd
@@ -601,6 +618,7 @@ def submit_module_build(db_session, username, stream_or_packager, params, module
validate_mmd(mmd)
_apply_dep_overrides(mmd, params)
_apply_side_tag(mmd, params)
_apply_rpm_component_ref_overrides(mmd, params)
_modify_buildtime_streams(db_session, mmd, resolve_base_module_virtual_streams)
_process_support_streams(db_session, mmd, params)
mmds += generate_expanded_mmds(db_session, mmd, raise_if_stream_ambigous,

21
module_build_service/web/views.py
View File

@@ -369,6 +369,7 @@ class BaseHandler(object):
valid_params = {
"branch",
"buildrequire_overrides",
"rpm_component_ref_overrides",
"modulemd",
"module_name",
"module_stream",
@@ -425,6 +426,25 @@ class BaseHandler(object):
if not isinstance(stream, string_types):
raise ValidationError(invalid_override_msg)
def _validate_ref_overrides_format(self, key):
"""
Validate any dependency overrides provided to the API.
:param str key: the override key to validate
:raises ValidationError: when the overrides are an invalid format
"""
if not self.data.get(key):
return
invalid_override_msg = (
'The "{}" parameter must be an object with the keys as component '
"names and the values as strings of refs".format(key)
)
if not isinstance(self.data[key], dict):
raise ValidationError(invalid_override_msg)
for ref in self.data[key].values():
if not isinstance(ref, string_types):
raise ValidationError(invalid_override_msg)
def validate_optional_params(self):
forbidden_params = [k for k in self.data if k not in self.valid_params]
if forbidden_params:
@@ -452,6 +472,7 @@ class BaseHandler(object):
self._validate_dep_overrides_format("buildrequire_overrides")
self._validate_dep_overrides_format("require_overrides")
self._validate_ref_overrides_format("rpm_component_ref_overrides")
if "reuse_components_from" in self.data:
if "rebuild_strategy" in self.data and self.data["rebuild_strategy"] == "all":

21
tests/test_scheduler/test_submit.py
View File

@@ -123,14 +123,22 @@ class TestSubmit:
None,
],
)
@pytest.mark.parametrize(
"rpm_component_ref_overrides",
[
{"tangerine": "ref_override"},
None,
],
)
@mock.patch("module_build_service.common.scm.SCM")
def test_format_mmd(self, mocked_scm, srpm_overrides, scmurl):
def test_format_mmd(self, mocked_scm, srpm_overrides, rpm_component_ref_overrides, scmurl):
mocked_scm.return_value.commit = "620ec77321b2ea7b0d67d82992dda3e1d67055b4"
# For all the RPMs in testmodule, get_latest is called
hashes_returned = {
"master": "fbed359411a1baa08d4a88e0d12d426fbf8f602c",
"f28": "4ceea43add2366d8b8c5a622a2fb563b625b9abf",
"f27": "5deef23acd2367d8b8d5a621a2fc568b695bc3bd",
"ref_override": "ref_override",
}
def mocked_get_latest(ref="master"):
@@ -146,6 +154,13 @@ class TestSubmit:
if srpm_overrides:
# Set a bogus ref that will raise an exception if not properly ignored.
mmd.get_rpm_component("perl-List-Compare").set_ref("bogus")
if rpm_component_ref_overrides:
xmd = mmd.get_xmd()
xmd["mbs"] = {"rpm_component_ref_overrides": rpm_component_ref_overrides}
mmd.set_xmd(xmd)
tangerine_ref = rpm_component_ref_overrides["tangerine"]
else:
tangerine_ref = hashes_returned["f27"]
format_mmd(mmd, scmurl, srpm_overrides=srpm_overrides)
# Make sure that original refs are not changed.
@@ -167,7 +182,7 @@ class TestSubmit:
"rpms": {
"perl-List-Compare": {"ref": "fbed359411a1baa08d4a88e0d12d426fbf8f602c"},
"perl-Tangerine": {"ref": "4ceea43add2366d8b8c5a622a2fb563b625b9abf"},
"tangerine": {"ref": "5deef23acd2367d8b8d5a621a2fc568b695bc3bd"},
"tangerine": {"ref": tangerine_ref},
},
"scmurl": "",
}
@@ -177,6 +192,8 @@ class TestSubmit:
xmd["mbs"]["scmurl"] = scmurl
if srpm_overrides:
xmd["mbs"]["rpms"]["perl-List-Compare"]["ref"] = match_anything
if rpm_component_ref_overrides:
xmd["mbs"]["rpm_component_ref_overrides"] = match_anything
mmd_xmd = mmd.get_xmd()
assert mmd_xmd == xmd

33
tests/test_web/test_views.py
View File

@@ -1583,6 +1583,9 @@ class TestSubmitBuild:
{"require_overrides": {"platform": "f28"}},
{"require_overrides": {"platform": 28}},
{"require_overrides": "platform:f28"},
{"rpm_component_ref_overrides": {"pkg_name": ["hash"]}},
{"rpm_component_ref_overrides": {"pkg_name": 28}},
{"rpm_component_ref_overrides": "pkg_name:hash"},
),
)
@patch("module_build_service.web.auth.get_user", return_value=user)
@@ -1614,8 +1617,38 @@ class TestSubmitBuild:
msg = msg.format("buildrequire_overrides")
elif "require_overrides" in optional_params:
msg = msg.format("require_overrides")
elif "rpm_component_ref_overrides" in optional_params:
msg = (
"The \"rpm_component_ref_overrides\" parameter must be an object with the keys "
"as component names and the values as strings of refs"
)
assert data == {"error": "Bad Request", "message": msg, "status": 400}
@patch("module_build_service.web.auth.get_user", return_value=user)
@patch("module_build_service.common.scm.SCM")
def test_submit_rpm_component_ref_overrides(self, mocked_scm, mocked_get_user):
optional_params = {"rpm_component_ref_overrides": {"pkg_name": "ref_override"}}
FakeSCM(
mocked_scm, "fakemodule", "fakemodule.yaml",
"3da541559918a808c2402bba5012f6c60b27661c")
post_url = "/module-build-service/2/module-builds/"
scm_url = (
"https://src.stg.fedoraproject.org/modules/testmodule.git?#"
"68931c90de214d9d13feefbd35246a81b6cb8d49"
)
json_input = {"branch": "master", "scmurl": scm_url}
json_input.update(optional_params)
rv = self.client.post(post_url, data=json.dumps(json_input))
data = json.loads(rv.data)
assert rv.status_code == 201
data = json.loads(rv.data)[0]
mmd = load_mmd(data["modulemd"])
xmd_override = mmd.get_xmd()["mbs"]["rpm_component_ref_overrides"]
assert xmd_override == optional_params["rpm_component_ref_overrides"]
@pytest.mark.parametrize("api_version", [1, 2])
@patch("module_build_service.web.auth.get_user", return_value=user)
@patch("module_build_service.common.scm.SCM")