mirror of
https://pagure.io/fm-orchestrator.git
synced 2026-04-14 14:59:44 +08:00
7761bbf5b4
Fix some oddities in the DBResolver implementation of get_compatible_base_module_modulemds() and make the MBSResolver version - which was previously just buggy - match that. (Tests for the MBSResolver version are added in a subsequent commit.) * If an empty virtual_streams argument was passed in, *all* streams were considered compatible. Throw an exception in this case - it should be considered an error. * If stream_version_lte=True, but the stream from the base module wasn't in the form FOOx.y.z, then throw an exception. This was previously treated like stream_version_lte=False, which is just a recipe for confusion and mistakes. test_get_reusable_module_use_latest_build() is rewritten to comprehensively test all possibilities, including the case that changed above. test_add_default_modules_compatible_platforms() is changed to run under allow_only_compatible_base_modules=False, since it expected Fedora-style virtual streams (versions not in FOOx.y.z form, all share the same stream), which doesn't make sense with allow_only_compatible_base_modules=True.
501 lines
22 KiB
Python
501 lines
22 KiB
Python
# -*- coding: utf-8 -*-
|
|
# SPDX-License-Identifier: MIT
|
|
from __future__ import absolute_import
|
|
|
|
from datetime import timedelta
|
|
|
|
import mock
|
|
import pytest
|
|
from sqlalchemy.orm.session import make_transient
|
|
|
|
from module_build_service.common import models
|
|
from module_build_service.common.errors import StreamNotXyz
|
|
from module_build_service.common.modulemd import Modulemd
|
|
from module_build_service.common.utils import import_mmd, load_mmd, mmd_to_str
|
|
from module_build_service.scheduler.db_session import db_session
|
|
from module_build_service.scheduler.reuse import get_reusable_component, get_reusable_module
|
|
from tests import read_staged_data
|
|
|
|
|
|
@pytest.mark.usefixtures("reuse_component_init_data")
|
|
class TestUtilsComponentReuse:
|
|
@pytest.mark.parametrize(
|
|
"changed_component", ["perl-List-Compare", "perl-Tangerine", "tangerine", None]
|
|
)
|
|
def test_get_reusable_component_different_component(self, changed_component):
|
|
second_module_build = models.ModuleBuild.get_by_id(db_session, 3)
|
|
if changed_component:
|
|
mmd = second_module_build.mmd()
|
|
mmd.get_rpm_component("tangerine").set_ref("00ea1da4192a2030f9ae023de3b3143ed647bbab")
|
|
second_module_build.modulemd = mmd_to_str(mmd)
|
|
|
|
second_module_changed_component = models.ComponentBuild.from_component_name(
|
|
db_session, changed_component, second_module_build.id)
|
|
second_module_changed_component.ref = "00ea1da4192a2030f9ae023de3b3143ed647bbab"
|
|
db_session.add(second_module_changed_component)
|
|
db_session.commit()
|
|
|
|
plc_rv = get_reusable_component(second_module_build, "perl-List-Compare")
|
|
pt_rv = get_reusable_component(second_module_build, "perl-Tangerine")
|
|
tangerine_rv = get_reusable_component(second_module_build, "tangerine")
|
|
|
|
if changed_component == "perl-List-Compare":
|
|
# perl-Tangerine can be reused even though a component in its batch has changed
|
|
assert plc_rv is None
|
|
assert pt_rv.package == "perl-Tangerine"
|
|
assert tangerine_rv is None
|
|
elif changed_component == "perl-Tangerine":
|
|
# perl-List-Compare can be reused even though a component in its batch has changed
|
|
assert plc_rv.package == "perl-List-Compare"
|
|
assert pt_rv is None
|
|
assert tangerine_rv is None
|
|
elif changed_component == "tangerine":
|
|
# perl-List-Compare and perl-Tangerine can be reused since they are in an earlier
|
|
# buildorder than tangerine
|
|
assert plc_rv.package == "perl-List-Compare"
|
|
assert pt_rv.package == "perl-Tangerine"
|
|
assert tangerine_rv is None
|
|
elif changed_component is None:
|
|
# Nothing has changed so everthing can be used
|
|
assert plc_rv.package == "perl-List-Compare"
|
|
assert pt_rv.package == "perl-Tangerine"
|
|
assert tangerine_rv.package == "tangerine"
|
|
|
|
def test_get_reusable_component_different_rpm_macros(self):
|
|
second_module_build = models.ModuleBuild.get_by_id(db_session, 3)
|
|
mmd = second_module_build.mmd()
|
|
buildopts = Modulemd.Buildopts()
|
|
buildopts.set_rpm_macros("%my_macro 1")
|
|
mmd.set_buildopts(buildopts)
|
|
second_module_build.modulemd = mmd_to_str(mmd)
|
|
db_session.commit()
|
|
|
|
plc_rv = get_reusable_component(second_module_build, "perl-List-Compare")
|
|
assert plc_rv is None
|
|
|
|
pt_rv = get_reusable_component(second_module_build, "perl-Tangerine")
|
|
assert pt_rv is None
|
|
|
|
@pytest.mark.parametrize("set_current_arch", [True, False])
|
|
@pytest.mark.parametrize("set_database_arch", [True, False])
|
|
def test_get_reusable_component_different_arches(
|
|
self, set_database_arch, set_current_arch
|
|
):
|
|
second_module_build = models.ModuleBuild.get_by_id(db_session, 3)
|
|
|
|
if set_current_arch: # set architecture for current build
|
|
mmd = second_module_build.mmd()
|
|
component = mmd.get_rpm_component("tangerine")
|
|
component.reset_arches()
|
|
component.add_restricted_arch("i686")
|
|
second_module_build.modulemd = mmd_to_str(mmd)
|
|
db_session.commit()
|
|
|
|
if set_database_arch: # set architecture for build in database
|
|
second_module_changed_component = models.ComponentBuild.from_component_name(
|
|
db_session, "tangerine", 2)
|
|
mmd = second_module_changed_component.module_build.mmd()
|
|
component = mmd.get_rpm_component("tangerine")
|
|
component.reset_arches()
|
|
component.add_restricted_arch("i686")
|
|
second_module_changed_component.module_build.modulemd = mmd_to_str(mmd)
|
|
db_session.commit()
|
|
|
|
tangerine = get_reusable_component(second_module_build, "tangerine")
|
|
assert bool(tangerine is None) != bool(set_current_arch == set_database_arch)
|
|
|
|
@pytest.mark.parametrize(
|
|
"reuse_component",
|
|
["perl-Tangerine", "perl-List-Compare", "tangerine"])
|
|
@pytest.mark.parametrize(
|
|
"changed_component",
|
|
["perl-Tangerine", "perl-List-Compare", "tangerine"])
|
|
def test_get_reusable_component_different_batch(
|
|
self, changed_component, reuse_component
|
|
):
|
|
"""
|
|
Test that we get the correct reuse behavior for the changed-and-after strategy. Changes
|
|
to earlier batches should prevent reuse, but changes to later batches should not.
|
|
For context, see https://pagure.io/fm-orchestrator/issue/1298
|
|
"""
|
|
|
|
if changed_component == reuse_component:
|
|
# we're only testing the cases where these are different
|
|
# this case is already covered by test_get_reusable_component_different_component
|
|
return
|
|
|
|
second_module_build = models.ModuleBuild.get_by_id(db_session, 3)
|
|
|
|
# update batch for changed component
|
|
changed_component = models.ComponentBuild.from_component_name(
|
|
db_session, changed_component, second_module_build.id)
|
|
orig_batch = changed_component.batch
|
|
changed_component.batch = orig_batch + 1
|
|
db_session.commit()
|
|
|
|
reuse_component = models.ComponentBuild.from_component_name(
|
|
db_session, reuse_component, second_module_build.id)
|
|
|
|
reuse_result = get_reusable_component(second_module_build, reuse_component.package)
|
|
# Component reuse should only be blocked when an earlier batch has been changed.
|
|
# In this case, orig_batch is the earliest batch that has been changed (the changed
|
|
# component has been removed from it and added to the following one).
|
|
assert bool(reuse_result is None) == bool(reuse_component.batch > orig_batch)
|
|
|
|
@pytest.mark.parametrize(
|
|
"reuse_component",
|
|
["perl-Tangerine", "perl-List-Compare", "tangerine"])
|
|
@pytest.mark.parametrize(
|
|
"changed_component",
|
|
["perl-Tangerine", "perl-List-Compare", "tangerine"])
|
|
def test_get_reusable_component_different_arch_in_batch(
|
|
self, changed_component, reuse_component
|
|
):
|
|
"""
|
|
Test that we get the correct reuse behavior for the changed-and-after strategy. Changes
|
|
to the architectures in earlier batches should prevent reuse, but such changes to later
|
|
batches should not.
|
|
For context, see https://pagure.io/fm-orchestrator/issue/1298
|
|
"""
|
|
if changed_component == reuse_component:
|
|
# we're only testing the cases where these are different
|
|
# this case is already covered by test_get_reusable_component_different_arches
|
|
return
|
|
|
|
second_module_build = models.ModuleBuild.get_by_id(db_session, 3)
|
|
|
|
# update arch for changed component
|
|
mmd = second_module_build.mmd()
|
|
component = mmd.get_rpm_component(changed_component)
|
|
component.reset_arches()
|
|
component.add_restricted_arch("i686")
|
|
second_module_build.modulemd = mmd_to_str(mmd)
|
|
db_session.commit()
|
|
|
|
changed_component = models.ComponentBuild.from_component_name(
|
|
db_session, changed_component, second_module_build.id)
|
|
reuse_component = models.ComponentBuild.from_component_name(
|
|
db_session, reuse_component, second_module_build.id)
|
|
|
|
reuse_result = get_reusable_component(second_module_build, reuse_component.package)
|
|
# Changing the arch of a component should prevent reuse only when the changed component
|
|
# is in a batch earlier than the component being considered for reuse.
|
|
assert bool(reuse_result is None) == bool(reuse_component.batch > changed_component.batch)
|
|
|
|
@pytest.mark.parametrize("rebuild_strategy", models.ModuleBuild.rebuild_strategies.keys())
|
|
def test_get_reusable_component_different_buildrequires_stream(self, rebuild_strategy):
|
|
first_module_build = models.ModuleBuild.get_by_id(db_session, 2)
|
|
first_module_build.rebuild_strategy = rebuild_strategy
|
|
db_session.commit()
|
|
|
|
second_module_build = models.ModuleBuild.get_by_id(db_session, 3)
|
|
mmd = second_module_build.mmd()
|
|
xmd = mmd.get_xmd()
|
|
xmd["mbs"]["buildrequires"]["platform"]["stream"] = "different"
|
|
deps = Modulemd.Dependencies()
|
|
deps.add_buildtime_stream("platform", "different")
|
|
deps.add_runtime_stream("platform", "different")
|
|
mmd.clear_dependencies()
|
|
mmd.add_dependencies(deps)
|
|
|
|
mmd.set_xmd(xmd)
|
|
second_module_build.modulemd = mmd_to_str(mmd)
|
|
second_module_build.build_context = \
|
|
models.ModuleBuild.contexts_from_mmd(second_module_build.modulemd).build_context
|
|
second_module_build.rebuild_strategy = rebuild_strategy
|
|
db_session.commit()
|
|
|
|
plc_rv = get_reusable_component(second_module_build, "perl-List-Compare")
|
|
pt_rv = get_reusable_component(second_module_build, "perl-Tangerine")
|
|
tangerine_rv = get_reusable_component(second_module_build, "tangerine")
|
|
|
|
assert plc_rv is None
|
|
assert pt_rv is None
|
|
assert tangerine_rv is None
|
|
|
|
def test_get_reusable_component_different_buildrequires(self):
|
|
second_module_build = models.ModuleBuild.get_by_id(db_session, 3)
|
|
mmd = second_module_build.mmd()
|
|
mmd.get_dependencies()[0].add_buildtime_stream("some_module", "master")
|
|
xmd = mmd.get_xmd()
|
|
xmd["mbs"]["buildrequires"] = {
|
|
"some_module": {
|
|
"ref": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
|
|
"stream": "master",
|
|
"version": "20170123140147",
|
|
}
|
|
}
|
|
mmd.set_xmd(xmd)
|
|
second_module_build.modulemd = mmd_to_str(mmd)
|
|
second_module_build.build_context = models.ModuleBuild.calculate_build_context(
|
|
xmd["mbs"]["buildrequires"])
|
|
db_session.commit()
|
|
|
|
plc_rv = get_reusable_component(second_module_build, "perl-List-Compare")
|
|
assert plc_rv is None
|
|
|
|
pt_rv = get_reusable_component(second_module_build, "perl-Tangerine")
|
|
assert pt_rv is None
|
|
|
|
tangerine_rv = get_reusable_component(second_module_build, "tangerine")
|
|
assert tangerine_rv is None
|
|
|
|
|
|
class TestReuseSharedUserSpace:
|
|
|
|
def test_get_reusable_component_shared_userspace_ordering(self,
|
|
require_platform_and_default_arch,
|
|
reuse_shared_userspace_init_data):
|
|
"""
|
|
For modules with lot of components per batch, there is big chance that
|
|
the database will return them in different order than what we have for
|
|
current `new_module`. In this case, reuse code should still be able to
|
|
reuse the components.
|
|
"""
|
|
old_module = models.ModuleBuild.get_by_id(db_session, 2)
|
|
new_module = models.ModuleBuild.get_by_id(db_session, 3)
|
|
rv = get_reusable_component(new_module, "llvm", previous_module_build=old_module)
|
|
assert rv.package == "llvm"
|
|
|
|
|
|
@pytest.mark.usefixtures("reuse_component_init_data")
|
|
class TestUtilsModuleReuse:
|
|
|
|
def test_get_reusable_module_when_reused_module_not_set(self):
|
|
module = db_session.query(models.ModuleBuild)\
|
|
.filter_by(name="testmodule")\
|
|
.order_by(models.ModuleBuild.id.desc())\
|
|
.first()
|
|
module.state = models.BUILD_STATES["build"]
|
|
db_session.commit()
|
|
|
|
assert not module.reused_module
|
|
|
|
reusable_module = get_reusable_module(module)
|
|
|
|
assert module.reused_module
|
|
assert reusable_module.id == module.reused_module_id
|
|
|
|
def test_get_reusable_module_when_reused_module_already_set(self):
|
|
modules = db_session.query(models.ModuleBuild)\
|
|
.filter_by(name="testmodule")\
|
|
.order_by(models.ModuleBuild.id.desc())\
|
|
.limit(2).all()
|
|
build_module = modules[0]
|
|
reused_module = modules[1]
|
|
build_module.state = models.BUILD_STATES["build"]
|
|
build_module.reused_module_id = reused_module.id
|
|
db_session.commit()
|
|
|
|
assert build_module.reused_module
|
|
assert reused_module == build_module.reused_module
|
|
|
|
reusable_module = get_reusable_module(build_module)
|
|
|
|
assert build_module.reused_module
|
|
assert reusable_module.id == build_module.reused_module_id
|
|
assert reusable_module.id == reused_module.id
|
|
|
|
_EXCEPTION = object()
|
|
|
|
@pytest.mark.parametrize("allow_ocbm,build_req,available_req,expected", [
|
|
# When allow_only_compatible_base_modules is false, only modules
|
|
# built against the exact same base module stream can be used
|
|
(False, "f29.2.0", ("f29.2.0",), "f29.2.0"),
|
|
(False, "f29.2.0", ("f29.0.0",), None),
|
|
# When it is true, any base module x.y2.z2 where y2.z2 <= y1.z1
|
|
# is a candidate
|
|
(True, "f29.2.0", ("f29.2.0", "f29.0.0", "f29.3.0"), "f29.2.0"),
|
|
(True, "f29.2.0", ("f29.1.0", "f29.0.0", "f29.3.0"), "f29.1.0"),
|
|
# But if the major is different, they don't (the code below adds "f28.0.0"
|
|
# with the "f29" virtual stream, so it would be a candidate otherwise)
|
|
(True, "f29.2.0", ("f28.0.0",), None),
|
|
# the virtual stream must also match (+ is used to skip virtual stream addition)
|
|
(True, "f29.2.0", ("+f29.1.0",), None),
|
|
# If the buildrequired base module isn't in x.y.z form, we raise an
|
|
# exception
|
|
(True, "f29", ("f29",), _EXCEPTION),
|
|
(True, "f29", ("f29.0.0",), _EXCEPTION),
|
|
])
|
|
@mock.patch(
|
|
"module_build_service.common.config.Config.allow_only_compatible_base_modules",
|
|
new_callable=mock.PropertyMock,
|
|
)
|
|
def test_get_reusable_module_use_latest_build(
|
|
self, cfg, allow_ocbm, build_req, available_req, expected):
|
|
"""
|
|
Test that the `get_reusable_module` tries to reuse the latest module in case when
|
|
multiple modules can be reused allow_only_compatible_base_modules is True.
|
|
"""
|
|
cfg.return_value = allow_ocbm
|
|
|
|
# Create all the platform streams we need
|
|
needed_platform_streams = set([build_req])
|
|
needed_platform_streams.update(available_req)
|
|
platform_modules = {}
|
|
for stream in needed_platform_streams:
|
|
skip_virtual = False
|
|
if stream.startswith('+'):
|
|
skip_virtual = True
|
|
stream = stream[1:]
|
|
|
|
# Create platform:f29.0.0 with "f29" virtual stream.
|
|
mmd = load_mmd(read_staged_data("platform"))
|
|
mmd = mmd.copy("platform", stream)
|
|
xmd = mmd.get_xmd()
|
|
xmd["mbs"]["virtual_streams"] = [] if skip_virtual else ["f29"]
|
|
mmd.set_xmd(xmd)
|
|
platform_modules[stream] = import_mmd(db_session, mmd)[0]
|
|
|
|
# Modify a module to buildrequire a particular platform stream
|
|
def modify_buildrequires(module, platform_module):
|
|
mmd = module.mmd()
|
|
deps = mmd.get_dependencies()[0]
|
|
deps.clear_buildtime_dependencies()
|
|
deps.add_buildtime_stream('platform', platform_module.stream)
|
|
xmd = mmd.get_xmd()
|
|
xmd["mbs"]["buildrequires"]["platform"]["stream"] = platform_module.stream
|
|
mmd.set_xmd(xmd)
|
|
module.modulemd = mmd_to_str(mmd)
|
|
contexts = models.ModuleBuild.contexts_from_mmd(
|
|
module.modulemd
|
|
)
|
|
module.build_context = contexts.build_context
|
|
module.context = contexts.context
|
|
module.buildrequires = [platform_module]
|
|
|
|
stream_to_testmodule_id = {}
|
|
|
|
first = True
|
|
for stream in available_req:
|
|
if stream.startswith('+'):
|
|
stream = stream[1:]
|
|
|
|
if first:
|
|
# Reuse the testmodule:master already in the database
|
|
module = db_session.query(models.ModuleBuild).filter_by(
|
|
name="testmodule", state=models.BUILD_STATES["ready"]).one()
|
|
|
|
modify_buildrequires(module, platform_modules[stream])
|
|
time_completed = module.time_completed
|
|
|
|
first = False
|
|
else:
|
|
# Create another copy of `testmodule:master`, and modify it
|
|
module = db_session.query(models.ModuleBuild).filter_by(
|
|
name="testmodule", state=models.BUILD_STATES["ready"]).first()
|
|
|
|
# This is used to clone the ModuleBuild SQLAlchemy object without recreating it from
|
|
# scratch.
|
|
db_session.expunge(module)
|
|
make_transient(module)
|
|
|
|
modify_buildrequires(module, platform_modules[stream])
|
|
|
|
# Add modules with ascending time_completed
|
|
time_completed += timedelta(days=1)
|
|
module.time_completed = time_completed
|
|
|
|
# Set the `id` to None, so new one is generated by SQLAlchemy.
|
|
module.id = None
|
|
db_session.add(module)
|
|
|
|
db_session.commit()
|
|
stream_to_testmodule_id[stream] = module.id
|
|
|
|
module = db_session.query(models.ModuleBuild)\
|
|
.filter_by(name="testmodule")\
|
|
.filter_by(state=models.BUILD_STATES["build"])\
|
|
.one()
|
|
modify_buildrequires(module, platform_modules[build_req])
|
|
db_session.commit()
|
|
|
|
if expected is TestUtilsModuleReuse._EXCEPTION:
|
|
with pytest.raises(StreamNotXyz):
|
|
get_reusable_module(module)
|
|
else:
|
|
reusable_module = get_reusable_module(module)
|
|
if expected:
|
|
assert reusable_module.id == stream_to_testmodule_id[expected]
|
|
else:
|
|
assert reusable_module is None
|
|
|
|
@pytest.mark.parametrize("allow_ocbm", (True, False))
|
|
@mock.patch(
|
|
"module_build_service.common.config.Config.allow_only_compatible_base_modules",
|
|
new_callable=mock.PropertyMock,
|
|
)
|
|
@mock.patch("koji.ClientSession")
|
|
@mock.patch(
|
|
"module_build_service.common.config.Config.resolver",
|
|
new_callable=mock.PropertyMock, return_value="koji"
|
|
)
|
|
def test_get_reusable_module_koji_resolver(
|
|
self, resolver, ClientSession, cfg, allow_ocbm):
|
|
"""
|
|
Test that get_reusable_module works with KojiResolver.
|
|
"""
|
|
cfg.return_value = allow_ocbm
|
|
|
|
# Mock the listTagged so the testmodule:master is listed as tagged in the
|
|
# module-fedora-27-build Koji tag.
|
|
koji_session = ClientSession.return_value
|
|
koji_session.listTagged.return_value = [
|
|
{
|
|
"build_id": 123, "name": "testmodule", "version": "master",
|
|
"release": "20170109091357.78e4a6fd", "tag_name": "module-fedora-27-build"
|
|
}]
|
|
|
|
koji_session.multiCall.return_value = [
|
|
[build] for build in koji_session.listTagged.return_value]
|
|
|
|
# Mark platform:f28 as KojiResolver ready by defining "koji_tag_with_modules".
|
|
# Also define the "virtual_streams" to possibly confuse the get_reusable_module.
|
|
platform_f28 = db_session.query(models.ModuleBuild).filter_by(name="platform").one()
|
|
mmd = platform_f28.mmd()
|
|
xmd = mmd.get_xmd()
|
|
xmd["mbs"]["virtual_streams"] = ["fedora"]
|
|
xmd["mbs"]["koji_tag_with_modules"] = "module-fedora-27-build"
|
|
mmd.set_xmd(xmd)
|
|
platform_f28.modulemd = mmd_to_str(mmd)
|
|
platform_f28.update_virtual_streams(db_session, ["fedora"])
|
|
|
|
# Create platform:f27 without KojiResolver support.
|
|
mmd = load_mmd(read_staged_data("platform"))
|
|
mmd = mmd.copy("platform", "f27")
|
|
xmd = mmd.get_xmd()
|
|
xmd["mbs"]["virtual_streams"] = ["fedora"]
|
|
mmd.set_xmd(xmd)
|
|
platform_f27 = import_mmd(db_session, mmd)[0]
|
|
|
|
# Change the reusable testmodule:master to buildrequire platform:f27.
|
|
latest_module = db_session.query(models.ModuleBuild).filter_by(
|
|
name="testmodule", state=models.BUILD_STATES["ready"]).one()
|
|
mmd = latest_module.mmd()
|
|
xmd = mmd.get_xmd()
|
|
xmd["mbs"]["buildrequires"]["platform"]["stream"] = "f27"
|
|
mmd.set_xmd(xmd)
|
|
latest_module.modulemd = mmd_to_str(mmd)
|
|
latest_module.buildrequires = [platform_f27]
|
|
|
|
# Recompute the build_context and ensure that `build_context` changed while
|
|
# `build_context_no_bms` did not change.
|
|
contexts = models.ModuleBuild.contexts_from_mmd(latest_module.modulemd)
|
|
|
|
assert latest_module.build_context_no_bms == contexts.build_context_no_bms
|
|
assert latest_module.build_context != contexts.build_context
|
|
|
|
latest_module.build_context = contexts.build_context
|
|
latest_module.build_context_no_bms = contexts.build_context_no_bms
|
|
db_session.commit()
|
|
|
|
# Get the module we want to build.
|
|
module = db_session.query(models.ModuleBuild)\
|
|
.filter_by(name="testmodule")\
|
|
.filter_by(state=models.BUILD_STATES["build"])\
|
|
.one()
|
|
|
|
reusable_module = get_reusable_module(module)
|
|
|
|
assert reusable_module.id == latest_module.id
|