From 85e3a2c4964e5d7abf3fe576d760d7da41af4ed6 Mon Sep 17 00:00:00 2001 From: mprahl Date: Fri, 3 Jan 2020 11:50:07 -0500 Subject: [PATCH] Move utils/reuse.py to scheduler/reuse.py --- .../builder/KojiModuleBuilder.py | 2 +- module_build_service/scheduler/batches.py | 2 +- .../scheduler/handlers/modules.py | 2 +- .../{utils => scheduler}/reuse.py | 0 module_build_service/utils/__init__.py | 1 - tests/test_scheduler/test_reuse.py | 443 +++++++++++++++++ tests/test_utils/test_utils.py | 446 +----------------- 7 files changed, 453 insertions(+), 443 deletions(-) rename module_build_service/{utils => scheduler}/reuse.py (100%) create mode 100644 tests/test_scheduler/test_reuse.py diff --git a/module_build_service/builder/KojiModuleBuilder.py b/module_build_service/builder/KojiModuleBuilder.py index e310dbd1..974603ac 100644 --- a/module_build_service/builder/KojiModuleBuilder.py +++ b/module_build_service/builder/KojiModuleBuilder.py @@ -31,7 +31,7 @@ from module_build_service.common.koji import ( get_session, koji_multicall_map, koji_retrying_multicall_map, ) from module_build_service.scheduler import events -from module_build_service.utils import get_reusable_components, get_reusable_module +from module_build_service.scheduler.reuse import get_reusable_components, get_reusable_module logging.basicConfig(level=logging.DEBUG) diff --git a/module_build_service/scheduler/batches.py b/module_build_service/scheduler/batches.py index 59179c2b..4bcabbb1 100644 --- a/module_build_service/scheduler/batches.py +++ b/module_build_service/scheduler/batches.py @@ -6,7 +6,7 @@ import concurrent.futures from module_build_service import conf, log, models from module_build_service.db_session import db_session from module_build_service.scheduler import events -from module_build_service.utils.reuse import get_reusable_components, reuse_component +from module_build_service.scheduler.reuse import get_reusable_components, reuse_component def at_concurrent_component_threshold(config): diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index 572b7a4e..3ba5588f 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -10,7 +10,6 @@ from module_build_service.common.retry import retry import module_build_service.resolver import module_build_service.utils from module_build_service.utils import ( - attempt_to_reuse_all_components, record_component_builds, record_filtered_rpms, record_module_build_arches @@ -23,6 +22,7 @@ from module_build_service.scheduler.default_modules import ( from module_build_service.scheduler.greenwave import greenwave from module_build_service.utils.submit import format_mmd from module_build_service.scheduler import events +from module_build_service.scheduler.reuse import attempt_to_reuse_all_components from module_build_service.scheduler.ursine import handle_stream_collision_modules from requests.exceptions import ConnectionError diff --git a/module_build_service/utils/reuse.py b/module_build_service/scheduler/reuse.py similarity index 100% rename from module_build_service/utils/reuse.py rename to module_build_service/scheduler/reuse.py diff --git a/module_build_service/utils/__init__.py b/module_build_service/utils/__init__.py index 37d4849a..88410f49 100644 --- a/module_build_service/utils/__init__.py +++ b/module_build_service/utils/__init__.py @@ -1,4 +1,3 @@ # -*- coding: utf-8 -*- # SPDX-License-Identifier: MIT -from module_build_service.utils.reuse import * # noqa from module_build_service.utils.submit import * # noqa diff --git a/tests/test_scheduler/test_reuse.py b/tests/test_scheduler/test_reuse.py new file mode 100644 index 00000000..1d13ce67 --- /dev/null +++ b/tests/test_scheduler/test_reuse.py @@ -0,0 +1,443 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +import mock +import pytest +from sqlalchemy.orm.session import make_transient + +from module_build_service import models, Modulemd +from module_build_service.common.utils import import_mmd, load_mmd, mmd_to_str +from module_build_service.db_session import db_session +from module_build_service.scheduler.reuse import get_reusable_component, get_reusable_module +from tests import clean_database, read_staged_data + + +@pytest.mark.usefixtures("reuse_component_init_data") +class TestUtilsComponentReuse: + @pytest.mark.parametrize( + "changed_component", ["perl-List-Compare", "perl-Tangerine", "tangerine", None] + ) + def test_get_reusable_component_different_component(self, changed_component): + second_module_build = models.ModuleBuild.get_by_id(db_session, 3) + if changed_component: + mmd = second_module_build.mmd() + mmd.get_rpm_component("tangerine").set_ref("00ea1da4192a2030f9ae023de3b3143ed647bbab") + second_module_build.modulemd = mmd_to_str(mmd) + + second_module_changed_component = models.ComponentBuild.from_component_name( + db_session, changed_component, second_module_build.id) + second_module_changed_component.ref = "00ea1da4192a2030f9ae023de3b3143ed647bbab" + db_session.add(second_module_changed_component) + db_session.commit() + + plc_rv = get_reusable_component(second_module_build, "perl-List-Compare") + pt_rv = get_reusable_component(second_module_build, "perl-Tangerine") + tangerine_rv = get_reusable_component(second_module_build, "tangerine") + + if changed_component == "perl-List-Compare": + # perl-Tangerine can be reused even though a component in its batch has changed + assert plc_rv is None + assert pt_rv.package == "perl-Tangerine" + assert tangerine_rv is None + elif changed_component == "perl-Tangerine": + # perl-List-Compare can be reused even though a component in its batch has changed + assert plc_rv.package == "perl-List-Compare" + assert pt_rv is None + assert tangerine_rv is None + elif changed_component == "tangerine": + # perl-List-Compare and perl-Tangerine can be reused since they are in an earlier + # buildorder than tangerine + assert plc_rv.package == "perl-List-Compare" + assert pt_rv.package == "perl-Tangerine" + assert tangerine_rv is None + elif changed_component is None: + # Nothing has changed so everthing can be used + assert plc_rv.package == "perl-List-Compare" + assert pt_rv.package == "perl-Tangerine" + assert tangerine_rv.package == "tangerine" + + def test_get_reusable_component_different_rpm_macros(self): + second_module_build = models.ModuleBuild.get_by_id(db_session, 3) + mmd = second_module_build.mmd() + buildopts = Modulemd.Buildopts() + buildopts.set_rpm_macros("%my_macro 1") + mmd.set_buildopts(buildopts) + second_module_build.modulemd = mmd_to_str(mmd) + db_session.commit() + + plc_rv = get_reusable_component(second_module_build, "perl-List-Compare") + assert plc_rv is None + + pt_rv = get_reusable_component(second_module_build, "perl-Tangerine") + assert pt_rv is None + + @pytest.mark.parametrize("set_current_arch", [True, False]) + @pytest.mark.parametrize("set_database_arch", [True, False]) + def test_get_reusable_component_different_arches( + self, set_database_arch, set_current_arch + ): + second_module_build = models.ModuleBuild.get_by_id(db_session, 3) + + if set_current_arch: # set architecture for current build + mmd = second_module_build.mmd() + component = mmd.get_rpm_component("tangerine") + component.reset_arches() + component.add_restricted_arch("i686") + second_module_build.modulemd = mmd_to_str(mmd) + db_session.commit() + + if set_database_arch: # set architecture for build in database + second_module_changed_component = models.ComponentBuild.from_component_name( + db_session, "tangerine", 2) + mmd = second_module_changed_component.module_build.mmd() + component = mmd.get_rpm_component("tangerine") + component.reset_arches() + component.add_restricted_arch("i686") + second_module_changed_component.module_build.modulemd = mmd_to_str(mmd) + db_session.commit() + + tangerine = get_reusable_component(second_module_build, "tangerine") + assert bool(tangerine is None) != bool(set_current_arch == set_database_arch) + + @pytest.mark.parametrize( + "reuse_component", + ["perl-Tangerine", "perl-List-Compare", "tangerine"]) + @pytest.mark.parametrize( + "changed_component", + ["perl-Tangerine", "perl-List-Compare", "tangerine"]) + def test_get_reusable_component_different_batch( + self, changed_component, reuse_component + ): + """ + Test that we get the correct reuse behavior for the changed-and-after strategy. Changes + to earlier batches should prevent reuse, but changes to later batches should not. + For context, see https://pagure.io/fm-orchestrator/issue/1298 + """ + + if changed_component == reuse_component: + # we're only testing the cases where these are different + # this case is already covered by test_get_reusable_component_different_component + return + + second_module_build = models.ModuleBuild.get_by_id(db_session, 3) + + # update batch for changed component + changed_component = models.ComponentBuild.from_component_name( + db_session, changed_component, second_module_build.id) + orig_batch = changed_component.batch + changed_component.batch = orig_batch + 1 + db_session.commit() + + reuse_component = models.ComponentBuild.from_component_name( + db_session, reuse_component, second_module_build.id) + + reuse_result = get_reusable_component(second_module_build, reuse_component.package) + # Component reuse should only be blocked when an earlier batch has been changed. + # In this case, orig_batch is the earliest batch that has been changed (the changed + # component has been removed from it and added to the following one). + assert bool(reuse_result is None) == bool(reuse_component.batch > orig_batch) + + @pytest.mark.parametrize( + "reuse_component", + ["perl-Tangerine", "perl-List-Compare", "tangerine"]) + @pytest.mark.parametrize( + "changed_component", + ["perl-Tangerine", "perl-List-Compare", "tangerine"]) + def test_get_reusable_component_different_arch_in_batch( + self, changed_component, reuse_component + ): + """ + Test that we get the correct reuse behavior for the changed-and-after strategy. Changes + to the architectures in earlier batches should prevent reuse, but such changes to later + batches should not. + For context, see https://pagure.io/fm-orchestrator/issue/1298 + """ + if changed_component == reuse_component: + # we're only testing the cases where these are different + # this case is already covered by test_get_reusable_component_different_arches + return + + second_module_build = models.ModuleBuild.get_by_id(db_session, 3) + + # update arch for changed component + mmd = second_module_build.mmd() + component = mmd.get_rpm_component(changed_component) + component.reset_arches() + component.add_restricted_arch("i686") + second_module_build.modulemd = mmd_to_str(mmd) + db_session.commit() + + changed_component = models.ComponentBuild.from_component_name( + db_session, changed_component, second_module_build.id) + reuse_component = models.ComponentBuild.from_component_name( + db_session, reuse_component, second_module_build.id) + + reuse_result = get_reusable_component(second_module_build, reuse_component.package) + # Changing the arch of a component should prevent reuse only when the changed component + # is in a batch earlier than the component being considered for reuse. + assert bool(reuse_result is None) == bool(reuse_component.batch > changed_component.batch) + + @pytest.mark.parametrize("rebuild_strategy", models.ModuleBuild.rebuild_strategies.keys()) + def test_get_reusable_component_different_buildrequires_stream(self, rebuild_strategy): + first_module_build = models.ModuleBuild.get_by_id(db_session, 2) + first_module_build.rebuild_strategy = rebuild_strategy + db_session.commit() + + second_module_build = models.ModuleBuild.get_by_id(db_session, 3) + mmd = second_module_build.mmd() + xmd = mmd.get_xmd() + xmd["mbs"]["buildrequires"]["platform"]["stream"] = "different" + deps = Modulemd.Dependencies() + deps.add_buildtime_stream("platform", "different") + deps.add_runtime_stream("platform", "different") + mmd.clear_dependencies() + mmd.add_dependencies(deps) + + mmd.set_xmd(xmd) + second_module_build.modulemd = mmd_to_str(mmd) + second_module_build.build_context = \ + models.ModuleBuild.contexts_from_mmd(second_module_build.modulemd).build_context + second_module_build.rebuild_strategy = rebuild_strategy + db_session.commit() + + plc_rv = get_reusable_component(second_module_build, "perl-List-Compare") + pt_rv = get_reusable_component(second_module_build, "perl-Tangerine") + tangerine_rv = get_reusable_component(second_module_build, "tangerine") + + assert plc_rv is None + assert pt_rv is None + assert tangerine_rv is None + + def test_get_reusable_component_different_buildrequires(self): + second_module_build = models.ModuleBuild.get_by_id(db_session, 3) + mmd = second_module_build.mmd() + mmd.get_dependencies()[0].add_buildtime_stream("some_module", "master") + xmd = mmd.get_xmd() + xmd["mbs"]["buildrequires"] = { + "some_module": { + "ref": "da39a3ee5e6b4b0d3255bfef95601890afd80709", + "stream": "master", + "version": "20170123140147", + } + } + mmd.set_xmd(xmd) + second_module_build.modulemd = mmd_to_str(mmd) + second_module_build.build_context = models.ModuleBuild.calculate_build_context( + xmd["mbs"]["buildrequires"]) + db_session.commit() + + plc_rv = get_reusable_component(second_module_build, "perl-List-Compare") + assert plc_rv is None + + pt_rv = get_reusable_component(second_module_build, "perl-Tangerine") + assert pt_rv is None + + tangerine_rv = get_reusable_component(second_module_build, "tangerine") + assert tangerine_rv is None + + +class TestReuseSharedUserSpace: + def setup_method(self, test_method): + clean_database() + + def teardown_method(self, test_method): + clean_database() + + @pytest.mark.usefixtures("reuse_shared_userspace_init_data") + def test_get_reusable_component_shared_userspace_ordering(self): + """ + For modules with lot of components per batch, there is big chance that + the database will return them in different order than what we have for + current `new_module`. In this case, reuse code should still be able to + reuse the components. + """ + old_module = models.ModuleBuild.get_by_id(db_session, 2) + new_module = models.ModuleBuild.get_by_id(db_session, 3) + rv = get_reusable_component(new_module, "llvm", previous_module_build=old_module) + assert rv.package == "llvm" + + +@pytest.mark.usefixtures("reuse_component_init_data") +class TestUtilsModuleReuse: + + def test_get_reusable_module_when_reused_module_not_set(self): + module = db_session.query(models.ModuleBuild)\ + .filter_by(name="testmodule")\ + .order_by(models.ModuleBuild.id.desc())\ + .first() + module.state = models.BUILD_STATES["build"] + db_session.commit() + + assert not module.reused_module + + reusable_module = get_reusable_module(module) + + assert module.reused_module + assert reusable_module.id == module.reused_module_id + + def test_get_reusable_module_when_reused_module_already_set(self): + modules = db_session.query(models.ModuleBuild)\ + .filter_by(name="testmodule")\ + .order_by(models.ModuleBuild.id.desc())\ + .limit(2).all() + build_module = modules[0] + reused_module = modules[1] + build_module.state = models.BUILD_STATES["build"] + build_module.reused_module_id = reused_module.id + db_session.commit() + + assert build_module.reused_module + assert reused_module == build_module.reused_module + + reusable_module = get_reusable_module(build_module) + + assert build_module.reused_module + assert reusable_module.id == build_module.reused_module_id + assert reusable_module.id == reused_module.id + + @pytest.mark.parametrize("allow_ocbm", (True, False)) + @mock.patch( + "module_build_service.config.Config.allow_only_compatible_base_modules", + new_callable=mock.PropertyMock, + ) + def test_get_reusable_module_use_latest_build(self, cfg, allow_ocbm): + """ + Test that the `get_reusable_module` tries to reuse the latest module in case when + multiple modules can be reused allow_only_compatible_base_modules is True. + """ + cfg.return_value = allow_ocbm + # Set "fedora" virtual stream to platform:f28. + platform_f28 = db_session.query(models.ModuleBuild).filter_by(name="platform").one() + mmd = platform_f28.mmd() + xmd = mmd.get_xmd() + xmd["mbs"]["virtual_streams"] = ["fedora"] + mmd.set_xmd(xmd) + platform_f28.modulemd = mmd_to_str(mmd) + platform_f28.update_virtual_streams(db_session, ["fedora"]) + + # Create platform:f29 with "fedora" virtual stream. + mmd = load_mmd(read_staged_data("platform")) + mmd = mmd.copy("platform", "f29") + xmd = mmd.get_xmd() + xmd["mbs"]["virtual_streams"] = ["fedora"] + mmd.set_xmd(xmd) + platform_f29 = import_mmd(db_session, mmd)[0] + + # Create another copy of `testmodule:master` which should be reused, because its + # stream version will be higher than the previous one. Also set its buildrequires + # to platform:f29. + latest_module = db_session.query(models.ModuleBuild).filter_by( + name="testmodule", state=models.BUILD_STATES["ready"]).one() + # This is used to clone the ModuleBuild SQLAlchemy object without recreating it from + # scratch. + db_session.expunge(latest_module) + make_transient(latest_module) + + # Change the platform:f28 buildrequirement to platform:f29 and recompute the build_context. + mmd = latest_module.mmd() + xmd = mmd.get_xmd() + xmd["mbs"]["buildrequires"]["platform"]["stream"] = "f29" + mmd.set_xmd(xmd) + latest_module.modulemd = mmd_to_str(mmd) + latest_module.build_context = models.ModuleBuild.contexts_from_mmd( + latest_module.modulemd + ).build_context + latest_module.buildrequires = [platform_f29] + + # Set the `id` to None, so new one is generated by SQLAlchemy. + latest_module.id = None + db_session.add(latest_module) + db_session.commit() + + module = db_session.query(models.ModuleBuild)\ + .filter_by(name="testmodule")\ + .filter_by(state=models.BUILD_STATES["build"])\ + .one() + db_session.commit() + + reusable_module = get_reusable_module(module) + + if allow_ocbm: + assert reusable_module.id == latest_module.id + else: + first_module = db_session.query(models.ModuleBuild).filter_by( + name="testmodule", state=models.BUILD_STATES["ready"]).first() + assert reusable_module.id == first_module.id + + @pytest.mark.parametrize("allow_ocbm", (True, False)) + @mock.patch( + "module_build_service.config.Config.allow_only_compatible_base_modules", + new_callable=mock.PropertyMock, + ) + @mock.patch("koji.ClientSession") + @mock.patch( + "module_build_service.config.Config.resolver", + new_callable=mock.PropertyMock, return_value="koji" + ) + def test_get_reusable_module_koji_resolver( + self, resolver, ClientSession, cfg, allow_ocbm): + """ + Test that get_reusable_module works with KojiResolver. + """ + cfg.return_value = allow_ocbm + + # Mock the listTagged so the testmodule:master is listed as tagged in the + # module-fedora-27-build Koji tag. + koji_session = ClientSession.return_value + koji_session.listTagged.return_value = [ + { + "build_id": 123, "name": "testmodule", "version": "master", + "release": "20170109091357.78e4a6fd", "tag_name": "module-fedora-27-build" + }] + + koji_session.multiCall.return_value = [ + [build] for build in koji_session.listTagged.return_value] + + # Mark platform:f28 as KojiResolver ready by defining "koji_tag_with_modules". + # Also define the "virtual_streams" to possibly confuse the get_reusable_module. + platform_f28 = db_session.query(models.ModuleBuild).filter_by(name="platform").one() + mmd = platform_f28.mmd() + xmd = mmd.get_xmd() + xmd["mbs"]["virtual_streams"] = ["fedora"] + xmd["mbs"]["koji_tag_with_modules"] = "module-fedora-27-build" + mmd.set_xmd(xmd) + platform_f28.modulemd = mmd_to_str(mmd) + platform_f28.update_virtual_streams(db_session, ["fedora"]) + + # Create platform:f27 without KojiResolver support. + mmd = load_mmd(read_staged_data("platform")) + mmd = mmd.copy("platform", "f27") + xmd = mmd.get_xmd() + xmd["mbs"]["virtual_streams"] = ["fedora"] + mmd.set_xmd(xmd) + platform_f27 = import_mmd(db_session, mmd)[0] + + # Change the reusable testmodule:master to buildrequire platform:f27. + latest_module = db_session.query(models.ModuleBuild).filter_by( + name="testmodule", state=models.BUILD_STATES["ready"]).one() + mmd = latest_module.mmd() + xmd = mmd.get_xmd() + xmd["mbs"]["buildrequires"]["platform"]["stream"] = "f27" + mmd.set_xmd(xmd) + latest_module.modulemd = mmd_to_str(mmd) + latest_module.buildrequires = [platform_f27] + + # Recompute the build_context and ensure that `build_context` changed while + # `build_context_no_bms` did not change. + contexts = models.ModuleBuild.contexts_from_mmd(latest_module.modulemd) + + assert latest_module.build_context_no_bms == contexts.build_context_no_bms + assert latest_module.build_context != contexts.build_context + + latest_module.build_context = contexts.build_context + latest_module.build_context_no_bms = contexts.build_context_no_bms + db_session.commit() + + # Get the module we want to build. + module = db_session.query(models.ModuleBuild)\ + .filter_by(name="testmodule")\ + .filter_by(state=models.BUILD_STATES["build"])\ + .one() + + reusable_module = get_reusable_module(module) + + assert reusable_module.id == latest_module.id diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index 66c19d01..8e6489ab 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -8,14 +8,12 @@ from shutil import copyfile, rmtree from datetime import datetime from werkzeug.datastructures import FileStorage from mock import patch -from sqlalchemy.orm.session import make_transient -from module_build_service.common.utils import import_mmd, load_mmd, load_mmd_file, mmd_to_str +from module_build_service.common.utils import load_mmd, load_mmd_file, mmd_to_str import module_build_service.utils import module_build_service.scm -from module_build_service import models, conf +from module_build_service import app, models, conf from module_build_service.errors import ValidationError, UnprocessableEntity -from module_build_service.utils.reuse import get_reusable_module, get_reusable_component from module_build_service.utils.submit import format_mmd from tests import ( clean_database, @@ -29,7 +27,6 @@ import pytest import module_build_service.scheduler.handlers.components from module_build_service.db_session import db_session from module_build_service.scheduler import events -from module_build_service import app, Modulemd BASE_DIR = path.abspath(path.dirname(__file__)) @@ -73,232 +70,12 @@ class FakeSCM(object): return commit_hash + sha1_hash[len(commit_hash):] -@pytest.mark.usefixtures("reuse_component_init_data") -class TestUtilsComponentReuse: - @pytest.mark.parametrize( - "changed_component", ["perl-List-Compare", "perl-Tangerine", "tangerine", None] - ) - def test_get_reusable_component_different_component(self, changed_component): - second_module_build = models.ModuleBuild.get_by_id(db_session, 3) - if changed_component: - mmd = second_module_build.mmd() - mmd.get_rpm_component("tangerine").set_ref("00ea1da4192a2030f9ae023de3b3143ed647bbab") - second_module_build.modulemd = mmd_to_str(mmd) +class TestUtils: + def setup_method(self, test_method): + clean_database() - second_module_changed_component = models.ComponentBuild.from_component_name( - db_session, changed_component, second_module_build.id) - second_module_changed_component.ref = "00ea1da4192a2030f9ae023de3b3143ed647bbab" - db_session.add(second_module_changed_component) - db_session.commit() - - plc_rv = get_reusable_component(second_module_build, "perl-List-Compare") - pt_rv = get_reusable_component(second_module_build, "perl-Tangerine") - tangerine_rv = get_reusable_component(second_module_build, "tangerine") - - if changed_component == "perl-List-Compare": - # perl-Tangerine can be reused even though a component in its batch has changed - assert plc_rv is None - assert pt_rv.package == "perl-Tangerine" - assert tangerine_rv is None - elif changed_component == "perl-Tangerine": - # perl-List-Compare can be reused even though a component in its batch has changed - assert plc_rv.package == "perl-List-Compare" - assert pt_rv is None - assert tangerine_rv is None - elif changed_component == "tangerine": - # perl-List-Compare and perl-Tangerine can be reused since they are in an earlier - # buildorder than tangerine - assert plc_rv.package == "perl-List-Compare" - assert pt_rv.package == "perl-Tangerine" - assert tangerine_rv is None - elif changed_component is None: - # Nothing has changed so everthing can be used - assert plc_rv.package == "perl-List-Compare" - assert pt_rv.package == "perl-Tangerine" - assert tangerine_rv.package == "tangerine" - - def test_get_reusable_component_different_rpm_macros(self): - second_module_build = models.ModuleBuild.get_by_id(db_session, 3) - mmd = second_module_build.mmd() - buildopts = Modulemd.Buildopts() - buildopts.set_rpm_macros("%my_macro 1") - mmd.set_buildopts(buildopts) - second_module_build.modulemd = mmd_to_str(mmd) - db_session.commit() - - plc_rv = get_reusable_component(second_module_build, "perl-List-Compare") - assert plc_rv is None - - pt_rv = get_reusable_component(second_module_build, "perl-Tangerine") - assert pt_rv is None - - @pytest.mark.parametrize("set_current_arch", [True, False]) - @pytest.mark.parametrize("set_database_arch", [True, False]) - def test_get_reusable_component_different_arches( - self, set_database_arch, set_current_arch - ): - second_module_build = models.ModuleBuild.get_by_id(db_session, 3) - - if set_current_arch: # set architecture for current build - mmd = second_module_build.mmd() - component = mmd.get_rpm_component("tangerine") - component.reset_arches() - component.add_restricted_arch("i686") - second_module_build.modulemd = mmd_to_str(mmd) - db_session.commit() - - if set_database_arch: # set architecture for build in database - second_module_changed_component = models.ComponentBuild.from_component_name( - db_session, "tangerine", 2) - mmd = second_module_changed_component.module_build.mmd() - component = mmd.get_rpm_component("tangerine") - component.reset_arches() - component.add_restricted_arch("i686") - second_module_changed_component.module_build.modulemd = mmd_to_str(mmd) - db_session.commit() - - tangerine = get_reusable_component(second_module_build, "tangerine") - assert bool(tangerine is None) != bool(set_current_arch == set_database_arch) - - @pytest.mark.parametrize( - "reuse_component", - ["perl-Tangerine", "perl-List-Compare", "tangerine"]) - @pytest.mark.parametrize( - "changed_component", - ["perl-Tangerine", "perl-List-Compare", "tangerine"]) - def test_get_reusable_component_different_batch( - self, changed_component, reuse_component - ): - """ - Test that we get the correct reuse behavior for the changed-and-after strategy. Changes - to earlier batches should prevent reuse, but changes to later batches should not. - For context, see https://pagure.io/fm-orchestrator/issue/1298 - """ - - if changed_component == reuse_component: - # we're only testing the cases where these are different - # this case is already covered by test_get_reusable_component_different_component - return - - second_module_build = models.ModuleBuild.get_by_id(db_session, 3) - - # update batch for changed component - changed_component = models.ComponentBuild.from_component_name( - db_session, changed_component, second_module_build.id) - orig_batch = changed_component.batch - changed_component.batch = orig_batch + 1 - db_session.commit() - - reuse_component = models.ComponentBuild.from_component_name( - db_session, reuse_component, second_module_build.id) - - reuse_result = module_build_service.utils.get_reusable_component( - second_module_build, reuse_component.package) - # Component reuse should only be blocked when an earlier batch has been changed. - # In this case, orig_batch is the earliest batch that has been changed (the changed - # component has been removed from it and added to the following one). - assert bool(reuse_result is None) == bool(reuse_component.batch > orig_batch) - - @pytest.mark.parametrize( - "reuse_component", - ["perl-Tangerine", "perl-List-Compare", "tangerine"]) - @pytest.mark.parametrize( - "changed_component", - ["perl-Tangerine", "perl-List-Compare", "tangerine"]) - def test_get_reusable_component_different_arch_in_batch( - self, changed_component, reuse_component - ): - """ - Test that we get the correct reuse behavior for the changed-and-after strategy. Changes - to the architectures in earlier batches should prevent reuse, but such changes to later - batches should not. - For context, see https://pagure.io/fm-orchestrator/issue/1298 - """ - if changed_component == reuse_component: - # we're only testing the cases where these are different - # this case is already covered by test_get_reusable_component_different_arches - return - - second_module_build = models.ModuleBuild.get_by_id(db_session, 3) - - # update arch for changed component - mmd = second_module_build.mmd() - component = mmd.get_rpm_component(changed_component) - component.reset_arches() - component.add_restricted_arch("i686") - second_module_build.modulemd = mmd_to_str(mmd) - db_session.commit() - - changed_component = models.ComponentBuild.from_component_name( - db_session, changed_component, second_module_build.id) - reuse_component = models.ComponentBuild.from_component_name( - db_session, reuse_component, second_module_build.id) - - reuse_result = module_build_service.utils.get_reusable_component( - second_module_build, reuse_component.package) - # Changing the arch of a component should prevent reuse only when the changed component - # is in a batch earlier than the component being considered for reuse. - assert bool(reuse_result is None) == bool(reuse_component.batch > changed_component.batch) - - @pytest.mark.parametrize("rebuild_strategy", models.ModuleBuild.rebuild_strategies.keys()) - def test_get_reusable_component_different_buildrequires_stream(self, rebuild_strategy): - first_module_build = models.ModuleBuild.get_by_id(db_session, 2) - first_module_build.rebuild_strategy = rebuild_strategy - db_session.commit() - - second_module_build = models.ModuleBuild.get_by_id(db_session, 3) - mmd = second_module_build.mmd() - xmd = mmd.get_xmd() - xmd["mbs"]["buildrequires"]["platform"]["stream"] = "different" - deps = Modulemd.Dependencies() - deps.add_buildtime_stream("platform", "different") - deps.add_runtime_stream("platform", "different") - mmd.clear_dependencies() - mmd.add_dependencies(deps) - - mmd.set_xmd(xmd) - second_module_build.modulemd = mmd_to_str(mmd) - second_module_build.build_context = \ - module_build_service.models.ModuleBuild.contexts_from_mmd( - second_module_build.modulemd - ).build_context - second_module_build.rebuild_strategy = rebuild_strategy - db_session.commit() - - plc_rv = get_reusable_component(second_module_build, "perl-List-Compare") - pt_rv = get_reusable_component(second_module_build, "perl-Tangerine") - tangerine_rv = get_reusable_component(second_module_build, "tangerine") - - assert plc_rv is None - assert pt_rv is None - assert tangerine_rv is None - - def test_get_reusable_component_different_buildrequires(self): - second_module_build = models.ModuleBuild.get_by_id(db_session, 3) - mmd = second_module_build.mmd() - mmd.get_dependencies()[0].add_buildtime_stream("some_module", "master") - xmd = mmd.get_xmd() - xmd["mbs"]["buildrequires"] = { - "some_module": { - "ref": "da39a3ee5e6b4b0d3255bfef95601890afd80709", - "stream": "master", - "version": "20170123140147", - } - } - mmd.set_xmd(xmd) - second_module_build.modulemd = mmd_to_str(mmd) - second_module_build.build_context = models.ModuleBuild.calculate_build_context( - xmd["mbs"]["buildrequires"]) - db_session.commit() - - plc_rv = get_reusable_component(second_module_build, "perl-List-Compare") - assert plc_rv is None - - pt_rv = get_reusable_component(second_module_build, "perl-Tangerine") - assert pt_rv is None - - tangerine_rv = get_reusable_component(second_module_build, "tangerine") - assert tangerine_rv is None + def teardown_method(self, test_method): + clean_database() @patch("module_build_service.utils.submit.submit_module_build") def test_submit_module_build_from_yaml_with_skiptests(self, mock_submit): @@ -333,14 +110,6 @@ class TestUtilsComponentReuse: assert username_arg == username rmtree(module_dir) - -class TestUtils: - def setup_method(self, test_method): - clean_database() - - def teardown_method(self, test_method): - clean_database() - @patch("koji.ClientSession") def test_get_build_arches(self, ClientSession): session = ClientSession.return_value @@ -457,19 +226,6 @@ class TestUtils: mmd_xmd = mmd.get_xmd() assert mmd_xmd == xmd - @pytest.mark.usefixtures("reuse_shared_userspace_init_data") - def test_get_reusable_component_shared_userspace_ordering(self): - """ - For modules with lot of components per batch, there is big chance that - the database will return them in different order than what we have for - current `new_module`. In this case, reuse code should still be able to - reuse the components. - """ - old_module = models.ModuleBuild.get_by_id(db_session, 2) - new_module = models.ModuleBuild.get_by_id(db_session, 3) - rv = get_reusable_component(new_module, "llvm", previous_module_build=old_module) - assert rv.package == "llvm" - @patch("module_build_service.scm.SCM") def test_record_component_builds_duplicate_components(self, mocked_scm): # Mock for format_mmd to get components' latest ref @@ -822,191 +578,3 @@ class TestLocalBuilds: assert len(local_modules) == 1 assert local_modules[0].koji_tag.endswith("/module-platform-f28-3/results") - - -@pytest.mark.usefixtures("reuse_component_init_data") -class TestUtilsModuleReuse: - - def test_get_reusable_module_when_reused_module_not_set(self): - module = db_session.query(models.ModuleBuild)\ - .filter_by(name="testmodule")\ - .order_by(models.ModuleBuild.id.desc())\ - .first() - module.state = models.BUILD_STATES["build"] - db_session.commit() - - assert not module.reused_module - - reusable_module = get_reusable_module(module) - - assert module.reused_module - assert reusable_module.id == module.reused_module_id - - def test_get_reusable_module_when_reused_module_already_set(self): - modules = db_session.query(models.ModuleBuild)\ - .filter_by(name="testmodule")\ - .order_by(models.ModuleBuild.id.desc())\ - .limit(2).all() - build_module = modules[0] - reused_module = modules[1] - build_module.state = models.BUILD_STATES["build"] - build_module.reused_module_id = reused_module.id - db_session.commit() - - assert build_module.reused_module - assert reused_module == build_module.reused_module - - reusable_module = get_reusable_module(build_module) - - assert build_module.reused_module - assert reusable_module.id == build_module.reused_module_id - assert reusable_module.id == reused_module.id - - @pytest.mark.parametrize("allow_ocbm", (True, False)) - @patch( - "module_build_service.config.Config.allow_only_compatible_base_modules", - new_callable=mock.PropertyMock, - ) - def test_get_reusable_module_use_latest_build(self, cfg, allow_ocbm): - """ - Test that the `get_reusable_module` tries to reuse the latest module in case when - multiple modules can be reused allow_only_compatible_base_modules is True. - """ - cfg.return_value = allow_ocbm - # Set "fedora" virtual stream to platform:f28. - platform_f28 = db_session.query(models.ModuleBuild).filter_by(name="platform").one() - mmd = platform_f28.mmd() - xmd = mmd.get_xmd() - xmd["mbs"]["virtual_streams"] = ["fedora"] - mmd.set_xmd(xmd) - platform_f28.modulemd = mmd_to_str(mmd) - platform_f28.update_virtual_streams(db_session, ["fedora"]) - - # Create platform:f29 with "fedora" virtual stream. - mmd = load_mmd(read_staged_data("platform")) - mmd = mmd.copy("platform", "f29") - xmd = mmd.get_xmd() - xmd["mbs"]["virtual_streams"] = ["fedora"] - mmd.set_xmd(xmd) - platform_f29 = import_mmd(db_session, mmd)[0] - - # Create another copy of `testmodule:master` which should be reused, because its - # stream version will be higher than the previous one. Also set its buildrequires - # to platform:f29. - latest_module = db_session.query(models.ModuleBuild).filter_by( - name="testmodule", state=models.BUILD_STATES["ready"]).one() - # This is used to clone the ModuleBuild SQLAlchemy object without recreating it from - # scratch. - db_session.expunge(latest_module) - make_transient(latest_module) - - # Change the platform:f28 buildrequirement to platform:f29 and recompute the build_context. - mmd = latest_module.mmd() - xmd = mmd.get_xmd() - xmd["mbs"]["buildrequires"]["platform"]["stream"] = "f29" - mmd.set_xmd(xmd) - latest_module.modulemd = mmd_to_str(mmd) - latest_module.build_context = module_build_service.models.ModuleBuild.contexts_from_mmd( - latest_module.modulemd - ).build_context - latest_module.buildrequires = [platform_f29] - - # Set the `id` to None, so new one is generated by SQLAlchemy. - latest_module.id = None - db_session.add(latest_module) - db_session.commit() - - module = db_session.query(models.ModuleBuild)\ - .filter_by(name="testmodule")\ - .filter_by(state=models.BUILD_STATES["build"])\ - .one() - db_session.commit() - - reusable_module = get_reusable_module(module) - - if allow_ocbm: - assert reusable_module.id == latest_module.id - else: - first_module = db_session.query(models.ModuleBuild).filter_by( - name="testmodule", state=models.BUILD_STATES["ready"]).first() - assert reusable_module.id == first_module.id - - @pytest.mark.parametrize("allow_ocbm", (True, False)) - @patch( - "module_build_service.config.Config.allow_only_compatible_base_modules", - new_callable=mock.PropertyMock, - ) - @patch("koji.ClientSession") - @patch( - "module_build_service.config.Config.resolver", - new_callable=mock.PropertyMock, return_value="koji" - ) - def test_get_reusable_module_koji_resolver( - self, resolver, ClientSession, cfg, allow_ocbm): - """ - Test that get_reusable_module works with KojiResolver. - """ - cfg.return_value = allow_ocbm - - # Mock the listTagged so the testmodule:master is listed as tagged in the - # module-fedora-27-build Koji tag. - koji_session = ClientSession.return_value - koji_session.listTagged.return_value = [ - { - "build_id": 123, "name": "testmodule", "version": "master", - "release": "20170109091357.78e4a6fd", "tag_name": "module-fedora-27-build" - }] - - koji_session.multiCall.return_value = [ - [build] for build in koji_session.listTagged.return_value] - - # Mark platform:f28 as KojiResolver ready by defining "koji_tag_with_modules". - # Also define the "virtual_streams" to possibly confuse the get_reusable_module. - platform_f28 = db_session.query(models.ModuleBuild).filter_by(name="platform").one() - mmd = platform_f28.mmd() - xmd = mmd.get_xmd() - xmd["mbs"]["virtual_streams"] = ["fedora"] - xmd["mbs"]["koji_tag_with_modules"] = "module-fedora-27-build" - mmd.set_xmd(xmd) - platform_f28.modulemd = mmd_to_str(mmd) - platform_f28.update_virtual_streams(db_session, ["fedora"]) - - # Create platform:f27 without KojiResolver support. - mmd = load_mmd(read_staged_data("platform")) - mmd = mmd.copy("platform", "f27") - xmd = mmd.get_xmd() - xmd["mbs"]["virtual_streams"] = ["fedora"] - mmd.set_xmd(xmd) - platform_f27 = import_mmd(db_session, mmd)[0] - - # Change the reusable testmodule:master to buildrequire platform:f27. - latest_module = db_session.query(models.ModuleBuild).filter_by( - name="testmodule", state=models.BUILD_STATES["ready"]).one() - mmd = latest_module.mmd() - xmd = mmd.get_xmd() - xmd["mbs"]["buildrequires"]["platform"]["stream"] = "f27" - mmd.set_xmd(xmd) - latest_module.modulemd = mmd_to_str(mmd) - latest_module.buildrequires = [platform_f27] - - # Recompute the build_context and ensure that `build_context` changed while - # `build_context_no_bms` did not change. - contexts = module_build_service.models.ModuleBuild.contexts_from_mmd( - latest_module.modulemd) - - assert latest_module.build_context_no_bms == contexts.build_context_no_bms - assert latest_module.build_context != contexts.build_context - - latest_module.build_context = contexts.build_context - latest_module.build_context_no_bms = contexts.build_context_no_bms - db_session.commit() - - # Get the module we want to build. - module = db_session.query(models.ModuleBuild)\ - .filter_by(name="testmodule")\ - .filter_by(state=models.BUILD_STATES["build"])\ - .one() - - reusable_module = get_reusable_module(module) - - assert reusable_module.id == latest_module.id