modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-03 02:37:48 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Skip git ref checks for rpm components with srpm overrides

Browse Source 
Signed-off-by: Merlin Mathesius <mmathesi@redhat.com>
This commit is contained in:
Merlin Mathesius
2020-03-05 16:54:24 -06:00
committed by mprahl
parent ab90c7b710
commit de2a776226
3 changed files with 44 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
module_build_service/scheduler/handlers/modules.py
View File

@@ -31,7 +31,7 @@ from module_build_service.scheduler.default_modules import (
add_default_modules, handle_collisions_with_base_module_rpms)
from module_build_service.scheduler.greenwave import greenwave
from module_build_service.scheduler.reuse import attempt_to_reuse_all_components
from module_build_service.scheduler.submit import format_mmd
from module_build_service.scheduler.submit import format_mmd, get_module_srpm_overrides
from module_build_service.scheduler.ursine import handle_stream_collision_modules
logging.basicConfig(level=logging.DEBUG)
@@ -183,9 +183,11 @@ def init(msg_id, module_build_id, module_build_state):
arches = [arch.name for arch in build.arches]
defaults_added = add_default_modules(mmd)
# Get map of packages that have SRPM overrides
srpm_overrides = get_module_srpm_overrides(build)
# Format the modulemd by putting in defaults and replacing streams that
# are branches with commit hashes
format_mmd(mmd, build.scmurl, build, db_session)
format_mmd(mmd, build.scmurl, build, db_session, srpm_overrides)
record_component_builds(mmd, build)
# The ursine.handle_stream_collision_modules is Koji specific.

26
module_build_service/scheduler/submit.py
View File

@@ -182,7 +182,7 @@ def _scm_get_latest(pkg):
return {"pkg_name": pkg.get_name(), "pkg_ref": pkgref, "error": None}
def format_mmd(mmd, scmurl, module=None, db_session=None):
def format_mmd(mmd, scmurl, module=None, db_session=None, srpm_overrides=None):
"""
Prepares the modulemd for the MBS. This does things such as replacing the
branches of components with commit hashes and adding metadata in the xmd
@@ -192,7 +192,11 @@ def format_mmd(mmd, scmurl, module=None, db_session=None):
:param module: When specified together with `session`, the time_modified
of a module is updated regularly in case this method takes lot of time.
:param db_session: Database session to update the `module`.
:param dict srpm_overrides: Mapping of package names to SRPM links for all
component packages which have custom SRPM overrides specified.
"""
srpm_overrides = srpm_overrides or {}
xmd = mmd.get_xmd()
if "mbs" not in xmd:
xmd["mbs"] = {}
@@ -263,12 +267,18 @@ def format_mmd(mmd, scmurl, module=None, db_session=None):
pool = ThreadPool(20)
try:
# Filter out the packages which we have already resolved in possible
# previous runs of this method (can be caused by module build resubmition).
pkgs_to_resolve = [
mmd.get_rpm_component(name)
for name in mmd.get_rpm_component_names()
if name not in xmd["mbs"]["rpms"]
]
# previous runs of this method (can be caused by module build resubmition)
# or which have custom SRPMs and shouldn't be resolved.
pkgs_to_resolve = []
for name in mmd.get_rpm_component_names():
if name not in xmd["mbs"]["rpms"]:
if name in srpm_overrides:
# If this package has a custom SRPM, store an empty
# ref entry so no further verification takes place.
xmd["mbs"]["rpms"][name] = {"ref": None}
else:
pkgs_to_resolve.append(mmd.get_rpm_component(name))
async_result = pool.map_async(_scm_get_latest, pkgs_to_resolve)
# For modules with lot of components, the _scm_get_latest can take a lot of time.
@@ -436,7 +446,7 @@ def record_component_builds(
# It is OK to whitelist all URLs here, because the validity
# of every URL have been already checked in format_mmd(...).
included_mmd = fetch_mmd(full_url, whitelist_url=True)[0]
format_mmd(included_mmd, module.scmurl, module, db_session)
format_mmd(included_mmd, module.scmurl, module, db_session, srpm_overrides)
batch = record_component_builds(
included_mmd, module, batch, previous_buildorder, main_mmd)
continue

26
tests/test_scheduler/test_submit.py
View File

@@ -97,8 +97,15 @@ class TestSubmit:
None,
],
)
@pytest.mark.parametrize(
"srpm_overrides",
[
{"perl-List-Compare": "/path/to/perl-List-Compare.src.rpm"},
None,
],
)
@mock.patch("module_build_service.common.scm.SCM")
def test_format_mmd(self, mocked_scm, scmurl):
def test_format_mmd(self, mocked_scm, srpm_overrides, scmurl):
mocked_scm.return_value.commit = "620ec77321b2ea7b0d67d82992dda3e1d67055b4"
# For all the RPMs in testmodule, get_latest is called
hashes_returned = {
@@ -108,24 +115,33 @@ class TestSubmit:
}
def mocked_get_latest(ref="master"):
return hashes_returned[ref]
if ref in hashes_returned:
return hashes_returned[ref]
raise RuntimeError("ref %s not found." % ref)
mocked_scm.return_value.get_latest = mocked_get_latest
mmd = load_mmd(read_staged_data("testmodule"))
# Modify the component branches so we can identify them later on
mmd.get_rpm_component("perl-Tangerine").set_ref("f28")
mmd.get_rpm_component("tangerine").set_ref("f27")
format_mmd(mmd, scmurl)
if srpm_overrides:
# Set a bogus ref that will raise an exception if not properly ignored.
mmd.get_rpm_component("perl-List-Compare").set_ref("bogus")
format_mmd(mmd, scmurl, srpm_overrides=srpm_overrides)
# Make sure that original refs are not changed.
mmd_pkg_refs = [
mmd.get_rpm_component(pkg_name).get_ref()
for pkg_name in mmd.get_rpm_component_names()
]
assert set(mmd_pkg_refs) == set(hashes_returned.keys())
if srpm_overrides:
assert set(mmd_pkg_refs) == {'f27', 'f28', 'bogus'}
else:
assert set(mmd_pkg_refs) == {'f27', 'f28', 'master'}
deps = mmd.get_dependencies()[0]
assert deps.get_buildtime_modules() == ["platform"]
assert deps.get_buildtime_streams("platform") == ["f28"]
match_anything = type('eq_any', (), {"__eq__": lambda left, right: True})()
xmd = {
"mbs": {
"commit": "",
@@ -140,6 +156,8 @@ class TestSubmit:
if scmurl:
xmd["mbs"]["commit"] = "620ec77321b2ea7b0d67d82992dda3e1d67055b4"
xmd["mbs"]["scmurl"] = scmurl
if srpm_overrides:
xmd["mbs"]["rpms"]["perl-List-Compare"]["ref"] = match_anything
mmd_xmd = mmd.get_xmd()
assert mmd_xmd == xmd