mirror of
https://pagure.io/fm-orchestrator.git
synced 2026-02-02 20:59:06 +08:00
bfd9a13205
There is a need to rebuild the module builds done in CentOS 9 Stream internally in MBS to include them in RHEL. This is currenly a hard task, because the RPM components included in a module are usually taken from HEAD of the branch defined by their `ref` value. For the rebuild task, it means we would have to ensure that the HEAD of all RPM components points to right commit hash right before we start rebuilding CentOS 9 Stream module in internal MBS. This is very hard and fragile thing to do, especially if there are two different modules using the RPM component from the same branch. This is prone to race condition and makes the rebuilds quite complex and in some cases not possible to do without force pushes to RPM component repositories which is not acceptable by internal dist-git policy. This commit fixes it by allowing overriding the commit hash while submitting the module build. This helps in the mentioned situation, because we can keep internal RPM components branches in 1:1 sync with CentOS 9 Stream branches and HEAD can always point to the same commit in both internal and CentOS 9 Stream repositories. When the module rebuild is submitted in internal MBS, we can use this new feature to override the `ref` for each RPM component so it points to particular commit and the requirement for HEAD to point to this commit is no longer there. The `ref` is overriden only internally in MBS (but it is recorded in logs and in XMD section), so the input modulemd file is not altered. This is the same logic as used for other overrides (`buildrequire_overrides` or `side_tag`). This does not bring any security problem, because it is already possible to use commit hash in `ref`, so the package maintainer can already change the commit hash to any particular commit by using this `ref` value. Signed-off-by: Jan Kaluza <jkaluza@redhat.com>
387 lines
16 KiB
Python
387 lines
16 KiB
Python
# -*- coding: utf-8 -*-
|
|
# SPDX-License-Identifier: MIT
|
|
from __future__ import absolute_import
|
|
from datetime import datetime
|
|
|
|
import mock
|
|
import pytest
|
|
|
|
from module_build_service import app
|
|
from module_build_service.common import conf, models
|
|
from module_build_service.common.errors import UnprocessableEntity
|
|
from module_build_service.common.modulemd import Modulemd
|
|
from module_build_service.common.utils import load_mmd, load_mmd_file, mmd_to_str
|
|
from module_build_service.scheduler.db_session import db_session
|
|
import module_build_service.scheduler.handlers.components
|
|
from module_build_service.scheduler.submit import (
|
|
get_build_arches, format_mmd, record_component_builds, record_module_build_arches
|
|
)
|
|
from tests import (
|
|
read_staged_data,
|
|
staged_data_filename,
|
|
scheduler_init_data,
|
|
)
|
|
|
|
|
|
@pytest.mark.usefixtures("require_empty_database")
|
|
class TestSubmit:
|
|
|
|
@mock.patch("koji.ClientSession")
|
|
def test_get_build_arches(self, ClientSession, require_platform_and_default_arch):
|
|
session = ClientSession.return_value
|
|
session.getTag.return_value = {"arches": "ppc64le"}
|
|
mmd = load_mmd(read_staged_data("formatted_testmodule"))
|
|
r = get_build_arches(mmd, conf)
|
|
assert r == ["ppc64le"]
|
|
|
|
@mock.patch("koji.ClientSession")
|
|
def test_get_build_arches_no_arch_set(self, ClientSession):
|
|
"""
|
|
When no architecture is set in Koji tag, fallback to conf.arches.
|
|
"""
|
|
session = ClientSession.return_value
|
|
session.getTag.return_value = {"arches": ""}
|
|
mmd = load_mmd(read_staged_data("formatted_testmodule"))
|
|
r = get_build_arches(mmd, conf)
|
|
assert set(r) == set(conf.arches)
|
|
|
|
@mock.patch(
|
|
"module_build_service.common.config.Config.allowed_privileged_module_names",
|
|
new_callable=mock.PropertyMock,
|
|
return_value=["testmodule"],
|
|
)
|
|
def test_get_build_arches_koji_tag_arches(self, cfg):
|
|
mmd = load_mmd(read_staged_data("formatted_testmodule"))
|
|
xmd = mmd.get_xmd()
|
|
xmd["mbs"]["koji_tag_arches"] = ["ppc64", "ppc64le"]
|
|
mmd.set_xmd(xmd)
|
|
|
|
r = get_build_arches(mmd, conf)
|
|
assert r == ["ppc64", "ppc64le"]
|
|
|
|
@mock.patch.object(conf, "base_module_arches", new={"platform:xx": ["x86_64", "i686"]})
|
|
def test_get_build_arches_base_module_override(self):
|
|
mmd = load_mmd(read_staged_data("formatted_testmodule"))
|
|
xmd = mmd.get_xmd()
|
|
mbs_options = xmd["mbs"] if "mbs" in xmd.keys() else {}
|
|
mbs_options["buildrequires"] = {"platform": {"stream": "xx"}}
|
|
xmd["mbs"] = mbs_options
|
|
mmd.set_xmd(xmd)
|
|
|
|
r = get_build_arches(mmd, conf)
|
|
assert r == ["x86_64", "i686"]
|
|
|
|
@mock.patch.object(conf, "base_module_arches", new={"platform:xx": ["x86_64", "i686"]})
|
|
def test_get_build_arches_set_in_mmd(self):
|
|
mmd = load_mmd(read_staged_data("formatted_testmodule"))
|
|
xmd = mmd.get_xmd()
|
|
mbs_options = xmd.get("mbs", {})
|
|
mbs_options["buildrequires"] = {"platform": {"stream": "xx"}}
|
|
xmd["mbs"] = mbs_options
|
|
mmd.set_xmd(xmd)
|
|
try:
|
|
opts = Modulemd.Buildopts()
|
|
opts.add_arch("x86_64")
|
|
mmd.set_buildopts(opts)
|
|
expected_result = ["x86_64"]
|
|
except AttributeError:
|
|
# libmodulemd version < 2.8.3
|
|
expected_result = ["x86_64", "i686"]
|
|
|
|
r = get_build_arches(mmd, conf)
|
|
assert r == expected_result
|
|
|
|
@mock.patch("module_build_service.scheduler.submit.get_build_arches")
|
|
def test_record_module_build_arches(self, get_build_arches):
|
|
get_build_arches.return_value = ["x86_64", "i686"]
|
|
scheduler_init_data(1)
|
|
build = models.ModuleBuild.get_by_id(db_session, 2)
|
|
build.arches = []
|
|
record_module_build_arches(build.mmd(), build)
|
|
|
|
arches = {arch.name for arch in build.arches}
|
|
assert arches == set(get_build_arches.return_value)
|
|
|
|
# Ensure the function is idempotent
|
|
record_module_build_arches(build.mmd(), build)
|
|
assert len(build.arches) == len(get_build_arches.return_value)
|
|
|
|
@pytest.mark.parametrize(
|
|
"scmurl",
|
|
[
|
|
(
|
|
"https://src.stg.fedoraproject.org/modules/testmodule.git"
|
|
"?#620ec77321b2ea7b0d67d82992dda3e1d67055b4"
|
|
),
|
|
None,
|
|
],
|
|
)
|
|
@pytest.mark.parametrize(
|
|
"srpm_overrides",
|
|
[
|
|
{"perl-List-Compare": "/path/to/perl-List-Compare.src.rpm"},
|
|
None,
|
|
],
|
|
)
|
|
@pytest.mark.parametrize(
|
|
"rpm_component_ref_overrides",
|
|
[
|
|
{"tangerine": "ref_override"},
|
|
None,
|
|
],
|
|
)
|
|
@mock.patch("module_build_service.common.scm.SCM")
|
|
def test_format_mmd(self, mocked_scm, srpm_overrides, rpm_component_ref_overrides, scmurl):
|
|
mocked_scm.return_value.commit = "620ec77321b2ea7b0d67d82992dda3e1d67055b4"
|
|
# For all the RPMs in testmodule, get_latest is called
|
|
hashes_returned = {
|
|
"master": "fbed359411a1baa08d4a88e0d12d426fbf8f602c",
|
|
"f28": "4ceea43add2366d8b8c5a622a2fb563b625b9abf",
|
|
"f27": "5deef23acd2367d8b8d5a621a2fc568b695bc3bd",
|
|
"ref_override": "ref_override",
|
|
}
|
|
|
|
def mocked_get_latest(ref="master"):
|
|
if ref in hashes_returned:
|
|
return hashes_returned[ref]
|
|
raise RuntimeError("ref %s not found." % ref)
|
|
|
|
mocked_scm.return_value.get_latest = mocked_get_latest
|
|
mmd = load_mmd(read_staged_data("testmodule"))
|
|
# Modify the component branches so we can identify them later on
|
|
mmd.get_rpm_component("perl-Tangerine").set_ref("f28")
|
|
mmd.get_rpm_component("tangerine").set_ref("f27")
|
|
if srpm_overrides:
|
|
# Set a bogus ref that will raise an exception if not properly ignored.
|
|
mmd.get_rpm_component("perl-List-Compare").set_ref("bogus")
|
|
if rpm_component_ref_overrides:
|
|
xmd = mmd.get_xmd()
|
|
xmd["mbs"] = {"rpm_component_ref_overrides": rpm_component_ref_overrides}
|
|
mmd.set_xmd(xmd)
|
|
tangerine_ref = rpm_component_ref_overrides["tangerine"]
|
|
else:
|
|
tangerine_ref = hashes_returned["f27"]
|
|
format_mmd(mmd, scmurl, srpm_overrides=srpm_overrides)
|
|
|
|
# Make sure that original refs are not changed.
|
|
mmd_pkg_refs = [
|
|
mmd.get_rpm_component(pkg_name).get_ref()
|
|
for pkg_name in mmd.get_rpm_component_names()
|
|
]
|
|
if srpm_overrides:
|
|
assert set(mmd_pkg_refs) == {'f27', 'f28', 'bogus'}
|
|
else:
|
|
assert set(mmd_pkg_refs) == {'f27', 'f28', 'master'}
|
|
deps = mmd.get_dependencies()[0]
|
|
assert deps.get_buildtime_modules() == ["platform"]
|
|
assert deps.get_buildtime_streams("platform") == ["f28"]
|
|
match_anything = type('eq_any', (), {"__eq__": lambda left, right: True})()
|
|
xmd = {
|
|
"mbs": {
|
|
"commit": "",
|
|
"rpms": {
|
|
"perl-List-Compare": {"ref": "fbed359411a1baa08d4a88e0d12d426fbf8f602c"},
|
|
"perl-Tangerine": {"ref": "4ceea43add2366d8b8c5a622a2fb563b625b9abf"},
|
|
"tangerine": {"ref": tangerine_ref},
|
|
},
|
|
"scmurl": "",
|
|
}
|
|
}
|
|
if scmurl:
|
|
xmd["mbs"]["commit"] = "620ec77321b2ea7b0d67d82992dda3e1d67055b4"
|
|
xmd["mbs"]["scmurl"] = scmurl
|
|
if srpm_overrides:
|
|
xmd["mbs"]["rpms"]["perl-List-Compare"]["ref"] = match_anything
|
|
if rpm_component_ref_overrides:
|
|
xmd["mbs"]["rpm_component_ref_overrides"] = match_anything
|
|
mmd_xmd = mmd.get_xmd()
|
|
assert mmd_xmd == xmd
|
|
|
|
@mock.patch("module_build_service.common.scm.SCM")
|
|
def test_record_component_builds_duplicate_components(self, mocked_scm):
|
|
# Mock for format_mmd to get components' latest ref
|
|
mocked_scm.return_value.commit = "620ec77321b2ea7b0d67d82992dda3e1d67055b4"
|
|
mocked_scm.return_value.get_latest.side_effect = [
|
|
"4ceea43add2366d8b8c5a622a2fb563b625b9abf",
|
|
"fbed359411a1baa08d4a88e0d12d426fbf8f602c",
|
|
]
|
|
|
|
mmd = load_mmd(read_staged_data("testmodule"))
|
|
mmd = mmd.copy("testmodule-variant", "master")
|
|
module_build = module_build_service.common.models.ModuleBuild()
|
|
module_build.name = "testmodule-variant"
|
|
module_build.stream = "master"
|
|
module_build.version = 20170109091357
|
|
module_build.state = models.BUILD_STATES["init"]
|
|
module_build.scmurl = \
|
|
"https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79"
|
|
module_build.batch = 1
|
|
module_build.owner = "Tom Brady"
|
|
module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18)
|
|
module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35)
|
|
module_build.rebuild_strategy = "changed-and-after"
|
|
module_build.modulemd = mmd_to_str(mmd)
|
|
db_session.add(module_build)
|
|
db_session.commit()
|
|
# Rename the the modulemd to include
|
|
mmd = mmd.copy("testmodule")
|
|
# Remove perl-Tangerine and tangerine from the modulemd to include so only one
|
|
# component conflicts
|
|
mmd.remove_rpm_component("perl-Tangerine")
|
|
mmd.remove_rpm_component("tangerine")
|
|
|
|
error_msg = (
|
|
'The included module "testmodule" in "testmodule-variant" have '
|
|
"the following conflicting components: perl-List-Compare"
|
|
)
|
|
format_mmd(mmd, module_build.scmurl)
|
|
with pytest.raises(UnprocessableEntity) as e:
|
|
record_component_builds(mmd, module_build, main_mmd=module_build.mmd())
|
|
|
|
assert str(e.value) == error_msg
|
|
|
|
@mock.patch("module_build_service.common.scm.SCM")
|
|
def test_record_component_builds_set_weight(self, mocked_scm):
|
|
# Mock for format_mmd to get components' latest ref
|
|
mocked_scm.return_value.commit = "620ec77321b2ea7b0d67d82992dda3e1d67055b4"
|
|
mocked_scm.return_value.get_latest.side_effect = [
|
|
"4ceea43add2366d8b8c5a622a2fb563b625b9abf",
|
|
"fbed359411a1baa08d4a88e0d12d426fbf8f602c",
|
|
"dbed259411a1baa08d4a88e0d12d426fbf8f6037",
|
|
]
|
|
|
|
mmd = load_mmd(read_staged_data("testmodule"))
|
|
# Set the module name and stream
|
|
mmd = mmd.copy("testmodule", "master")
|
|
|
|
module_build = module_build_service.common.models.ModuleBuild()
|
|
module_build.name = "testmodule"
|
|
module_build.stream = "master"
|
|
module_build.version = 20170109091357
|
|
module_build.state = models.BUILD_STATES["init"]
|
|
module_build.scmurl = \
|
|
"https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79"
|
|
module_build.batch = 1
|
|
module_build.owner = "Tom Brady"
|
|
module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18)
|
|
module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35)
|
|
module_build.rebuild_strategy = "changed-and-after"
|
|
module_build.modulemd = mmd_to_str(mmd)
|
|
|
|
db_session.add(module_build)
|
|
db_session.commit()
|
|
|
|
format_mmd(mmd, module_build.scmurl)
|
|
record_component_builds(mmd, module_build)
|
|
db_session.commit()
|
|
|
|
assert module_build.state == models.BUILD_STATES["init"]
|
|
db_session.refresh(module_build)
|
|
for c in module_build.component_builds:
|
|
assert c.weight == 1.5
|
|
|
|
@mock.patch("module_build_service.common.scm.SCM")
|
|
def test_record_component_builds_component_exists_already(self, mocked_scm):
|
|
mocked_scm.return_value.commit = "620ec77321b2ea7b0d67d82992dda3e1d67055b4"
|
|
mocked_scm.return_value.get_latest.side_effect = [
|
|
"4ceea43add2366d8b8c5a622a2fb563b625b9abf",
|
|
"fbed359411a1baa08d4a88e0d12d426fbf8f602c",
|
|
"dbed259411a1baa08d4a88e0d12d426fbf8f6037",
|
|
|
|
"4ceea43add2366d8b8c5a622a2fb563b625b9abf",
|
|
# To simulate that when a module is resubmitted, some ref of
|
|
# its components is changed, which will cause MBS stops
|
|
# recording component to database and raise an error.
|
|
"abcdefg",
|
|
"dbed259411a1baa08d4a88e0d12d426fbf8f6037",
|
|
]
|
|
|
|
original_mmd = load_mmd(read_staged_data("testmodule"))
|
|
|
|
# Set the module name and stream
|
|
mmd = original_mmd.copy("testmodule", "master")
|
|
module_build = module_build_service.common.models.ModuleBuild()
|
|
module_build.name = "testmodule"
|
|
module_build.stream = "master"
|
|
module_build.version = 20170109091357
|
|
module_build.state = models.BUILD_STATES["init"]
|
|
module_build.scmurl = \
|
|
"https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79"
|
|
module_build.batch = 1
|
|
module_build.owner = "Tom Brady"
|
|
module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18)
|
|
module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35)
|
|
module_build.rebuild_strategy = "changed-and-after"
|
|
module_build.modulemd = mmd_to_str(mmd)
|
|
db_session.add(module_build)
|
|
db_session.commit()
|
|
|
|
format_mmd(mmd, module_build.scmurl)
|
|
record_component_builds(mmd, module_build)
|
|
db_session.commit()
|
|
|
|
mmd = original_mmd.copy("testmodule", "master")
|
|
|
|
from module_build_service.common.errors import ValidationError
|
|
with pytest.raises(
|
|
ValidationError,
|
|
match=r"Component build .+ of module build .+ already exists in database"):
|
|
format_mmd(mmd, module_build.scmurl)
|
|
record_component_builds(mmd, module_build)
|
|
|
|
@mock.patch("module_build_service.common.scm.SCM")
|
|
def test_format_mmd_arches(self, mocked_scm):
|
|
with app.app_context():
|
|
mocked_scm.return_value.commit = "620ec77321b2ea7b0d67d82992dda3e1d67055b4"
|
|
mocked_scm.return_value.get_latest.side_effect = [
|
|
"4ceea43add2366d8b8c5a622a2fb563b625b9abf",
|
|
"fbed359411a1baa08d4a88e0d12d426fbf8f602c",
|
|
"dbed259411a1baa08d4a88e0d12d426fbf8f6037",
|
|
"4ceea43add2366d8b8c5a622a2fb563b625b9abf",
|
|
"fbed359411a1baa08d4a88e0d12d426fbf8f602c",
|
|
"dbed259411a1baa08d4a88e0d12d426fbf8f6037",
|
|
]
|
|
|
|
testmodule_mmd_path = staged_data_filename("testmodule.yaml")
|
|
test_archs = ["powerpc", "i486"]
|
|
|
|
mmd1 = load_mmd_file(testmodule_mmd_path)
|
|
format_mmd(mmd1, None)
|
|
|
|
for pkg_name in mmd1.get_rpm_component_names():
|
|
pkg = mmd1.get_rpm_component(pkg_name)
|
|
assert set(pkg.get_arches()) == set(conf.arches)
|
|
|
|
mmd2 = load_mmd_file(testmodule_mmd_path)
|
|
|
|
for pkg_name in mmd2.get_rpm_component_names():
|
|
pkg = mmd2.get_rpm_component(pkg_name)
|
|
pkg.reset_arches()
|
|
for arch in test_archs:
|
|
pkg.add_restricted_arch(arch)
|
|
|
|
format_mmd(mmd2, None)
|
|
|
|
for pkg_name in mmd2.get_rpm_component_names():
|
|
pkg = mmd2.get_rpm_component(pkg_name)
|
|
assert set(pkg.get_arches()) == set(test_archs)
|
|
|
|
@mock.patch("module_build_service.common.scm.SCM")
|
|
@mock.patch("module_build_service.scheduler.submit.ThreadPool")
|
|
def test_format_mmd_update_time_modified(self, tp, mocked_scm, provide_test_data):
|
|
build = models.ModuleBuild.get_by_id(db_session, 2)
|
|
|
|
async_result = mock.MagicMock()
|
|
async_result.ready.side_effect = [False, False, False, True]
|
|
tp.return_value.map_async.return_value = async_result
|
|
|
|
test_datetime = datetime(2019, 2, 14, 11, 11, 45, 42968)
|
|
|
|
mmd = load_mmd(read_staged_data("testmodule"))
|
|
|
|
with mock.patch("module_build_service.scheduler.submit.datetime") as dt:
|
|
dt.utcnow.return_value = test_datetime
|
|
format_mmd(mmd, None, build, db_session)
|
|
|
|
assert build.time_modified == test_datetime
|