From 81ab984b1f78e1b4e68671c714683de67ff24ded Mon Sep 17 00:00:00 2001 From: mprahl Date: Thu, 2 Jan 2020 14:22:33 -0500 Subject: [PATCH] Move utils/batches.py to scheduler/batches.py --- docs/HOW_MBS_BUILDS_MODULES.rst | 2 +- .../{utils => scheduler}/batches.py | 4 +- .../scheduler/handlers/components.py | 2 +- .../scheduler/handlers/repos.py | 2 +- module_build_service/scheduler/producer.py | 11 +- module_build_service/utils/__init__.py | 1 - tests/test_build/test_build.py | 2 +- tests/test_scheduler/test_batches.py | 400 ++++++++++++++++++ tests/test_scheduler/test_poller.py | 4 +- tests/test_utils/test_utils.py | 396 ----------------- 10 files changed, 415 insertions(+), 409 deletions(-) rename module_build_service/{utils => scheduler}/batches.py (99%) create mode 100644 tests/test_scheduler/test_batches.py diff --git a/docs/HOW_MBS_BUILDS_MODULES.rst b/docs/HOW_MBS_BUILDS_MODULES.rst index 896499ee..7a845cf8 100644 --- a/docs/HOW_MBS_BUILDS_MODULES.rst +++ b/docs/HOW_MBS_BUILDS_MODULES.rst @@ -156,7 +156,7 @@ the ``scheduler.handlers.repos.done(...)`` method is called. This verifies that all the packages from the current batch (just module-build-macros for now) really appear in the generated repository and if so, it starts building the next batch by calling -``utils.batches.start_next_batch_build(...)``. +``module_build_service.scheduler.batches.start_next_batch_build(...)``. Building the next batch diff --git a/module_build_service/utils/batches.py b/module_build_service/scheduler/batches.py similarity index 99% rename from module_build_service/utils/batches.py rename to module_build_service/scheduler/batches.py index 3db48338..43140ff0 100644 --- a/module_build_service/utils/batches.py +++ b/module_build_service/scheduler/batches.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- # SPDX-License-Identifier: MIT -import threading import concurrent.futures +import threading from module_build_service import conf, log, models from module_build_service.db_session import db_session from module_build_service.scheduler import events -from .reuse import get_reusable_components, reuse_component +from module_build_service.utils.reuse import get_reusable_components, reuse_component def at_concurrent_component_threshold(config): diff --git a/module_build_service/scheduler/handlers/components.py b/module_build_service/scheduler/handlers/components.py index af93d559..d24966d2 100644 --- a/module_build_service/scheduler/handlers/components.py +++ b/module_build_service/scheduler/handlers/components.py @@ -11,7 +11,7 @@ from module_build_service.common.koji import get_session from module_build_service.utils.general import mmd_to_str from module_build_service.db_session import db_session from module_build_service.scheduler import events -from module_build_service.utils.batches import continue_batch_build +from module_build_service.scheduler.batches import continue_batch_build logging.basicConfig(level=logging.DEBUG) diff --git a/module_build_service/scheduler/handlers/repos.py b/module_build_service/scheduler/handlers/repos.py index e5f9029e..3f0fc1e7 100644 --- a/module_build_service/scheduler/handlers/repos.py +++ b/module_build_service/scheduler/handlers/repos.py @@ -6,7 +6,7 @@ import logging from datetime import datetime from module_build_service import celery_app, conf, models, log from module_build_service.builder import GenericBuilder -from module_build_service.utils import start_next_batch_build +from module_build_service.scheduler.batches import start_next_batch_build from module_build_service.db_session import db_session from module_build_service.scheduler import events diff --git a/module_build_service/scheduler/producer.py b/module_build_service/scheduler/producer.py index 0b50e255..6ac3a7c0 100644 --- a/module_build_service/scheduler/producer.py +++ b/module_build_service/scheduler/producer.py @@ -14,6 +14,10 @@ from module_build_service.builder import GenericBuilder from module_build_service.common.koji import get_session from module_build_service.utils.greenwave import greenwave from module_build_service.db_session import db_session +from module_build_service.scheduler.batches import ( + at_concurrent_component_threshold, + start_next_batch_build, +) from module_build_service.scheduler.consumer import ON_MODULE_CHANGE_HANDLERS from module_build_service.scheduler.handlers.components import build_task_finalize from module_build_service.scheduler.handlers.tags import tagged @@ -168,7 +172,7 @@ def fail_lost_builds(): @celery_app.task def process_paused_module_builds(): log.info("Looking for paused module builds in the build state") - if module_build_service.utils.at_concurrent_component_threshold(conf): + if at_concurrent_component_threshold(conf): log.debug( "Will not attempt to start paused module builds due to " "the concurrent build threshold being met" @@ -199,11 +203,10 @@ def process_paused_module_builds(): if has_missed_new_repo_message(module_build, builder.koji_session): log.info(" Processing the paused module build %r", module_build) - module_build_service.utils.start_next_batch_build( - conf, module_build, builder) + start_next_batch_build(conf, module_build, builder) # Check if we have met the threshold. - if module_build_service.utils.at_concurrent_component_threshold(conf): + if at_concurrent_component_threshold(conf): break diff --git a/module_build_service/utils/__init__.py b/module_build_service/utils/__init__.py index d91bf4ef..fe548da2 100644 --- a/module_build_service/utils/__init__.py +++ b/module_build_service/utils/__init__.py @@ -5,5 +5,4 @@ from module_build_service.utils.mse import * # noqa from module_build_service.utils.views import * # noqa from module_build_service.utils.reuse import * # noqa from module_build_service.utils.submit import * # noqa -from module_build_service.utils.batches import * # noqa from module_build_service.utils.ursine import * # noqa diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 66c9208b..4342a119 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -1789,7 +1789,7 @@ class TestBuild(BaseTestBuild): return result with patch( - "module_build_service.utils.batches.at_concurrent_component_threshold" + "module_build_service.scheduler.batches.at_concurrent_component_threshold" ) as mock_acct: # Once we get to batch 2, then simulate the concurrent threshold being met def _at_concurrent_component_threshold(config): diff --git a/tests/test_scheduler/test_batches.py b/tests/test_scheduler/test_batches.py new file mode 100644 index 00000000..9b722f2e --- /dev/null +++ b/tests/test_scheduler/test_batches.py @@ -0,0 +1,400 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +from mock import patch +from module_build_service import models, conf +import mock +import koji +import pytest +from module_build_service.db_session import db_session +from module_build_service.builder import GenericBuilder +from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder +from module_build_service.scheduler import events +from module_build_service.scheduler.batches import start_build_component, start_next_batch_build +from module_build_service.utils import validate_koji_tag + + +class DummyModuleBuilder(GenericBuilder): + """ + Dummy module builder + """ + + backend = "koji" + _build_id = 0 + + TAGGED_COMPONENTS = [] + + @validate_koji_tag("tag_name") + def __init__(self, db_session, owner, module, config, tag_name, components): + self.db_session = db_session + self.module_str = module + self.tag_name = tag_name + self.config = config + + def buildroot_connect(self, groups): + pass + + def buildroot_prep(self): + pass + + def buildroot_resume(self): + pass + + def buildroot_ready(self, artifacts=None): + return True + + def buildroot_add_dependency(self, dependencies): + pass + + def buildroot_add_artifacts(self, artifacts, install=False): + DummyModuleBuilder.TAGGED_COMPONENTS += artifacts + + def buildroot_add_repos(self, dependencies): + pass + + def tag_artifacts(self, artifacts): + pass + + def recover_orphaned_artifact(self, component_build): + return [] + + @property + def module_build_tag(self): + return {"name": self.tag_name + "-build"} + + def build(self, artifact_name, source): + DummyModuleBuilder._build_id += 1 + state = koji.BUILD_STATES["COMPLETE"] + reason = "Submitted %s to Koji" % (artifact_name) + return DummyModuleBuilder._build_id, state, reason, None + + @staticmethod + def get_disttag_srpm(disttag, module_build): + # @FIXME + return KojiModuleBuilder.get_disttag_srpm(disttag, module_build) + + def cancel_build(self, task_id): + pass + + def list_tasks_for_components(self, component_builds=None, state="active"): + pass + + def repo_from_tag(self, config, tag_name, arch): + pass + + def finalize(self, succeeded=True): + pass + + +@pytest.mark.usefixtures("reuse_component_init_data") +@patch( + "module_build_service.builder.GenericBuilder.default_buildroot_groups", + return_value={"build": [], "srpm-build": []}, +) +class TestBatches: + def setup_method(self, test_method): + GenericBuilder.register_backend_class(DummyModuleBuilder) + events.scheduler.reset() + + def teardown_method(self, test_method): + # clean_database() + DummyModuleBuilder.TAGGED_COMPONENTS = [] + GenericBuilder.register_backend_class(KojiModuleBuilder) + events.scheduler.reset() + + def test_start_next_batch_build_reuse(self, default_buildroot_groups): + """ + Tests that start_next_batch_build: + 1) Increments module.batch. + 2) Can reuse all components in batch + 3) Returns proper further_work messages for reused components. + 4) Returns the fake Repo change message + 5) Handling the further_work messages lead to proper tagging of + reused components. + """ + module_build = models.ModuleBuild.get_by_id(db_session, 3) + module_build.batch = 1 + + builder = mock.MagicMock() + builder.module_build_tag = {"name": "module-fedora-27-build"} + start_next_batch_build( + conf, module_build, builder) + + # Batch number should increase. + assert module_build.batch == 2 + + # buildsys.build.state.change messages in further_work should have + # build_new_state set to COMPLETE, but the current component build + # state should be set to BUILDING, so KojiBuildChange message handler + # handles the change properly. + for event in events.scheduler.queue: + event_info = event[3] + if event_info[0].startswith("reuse_component"): + assert event_info[2] == koji.BUILD_STATES["COMPLETE"] + component_build = models.ComponentBuild.from_component_event( + db_session, + task_id=event_info[1], + module_id=event_info[6]) + assert component_build.state == koji.BUILD_STATES["BUILDING"] + + # When we handle these KojiBuildChange messages, MBS should tag all + # the components just once. + events.scheduler.run() + + # Check that packages have been tagged just once. + assert len(DummyModuleBuilder.TAGGED_COMPONENTS) == 2 + + @patch("module_build_service.scheduler.batches.start_build_component") + def test_start_next_batch_build_reuse_some( + self, mock_sbc, default_buildroot_groups + ): + """ + Tests that start_next_batch_build: + 1) Increments module.batch. + 2) Can reuse all components in the batch that it can. + 3) Returns proper further_work messages for reused components. + 4) Builds the remaining components + 5) Handling the further_work messages lead to proper tagging of + reused components. + """ + module_build = models.ModuleBuild.get_by_id(db_session, 3) + module_build.batch = 1 + + plc_component = models.ComponentBuild.from_component_name( + db_session, "perl-List-Compare", 3) + plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd" + + builder = mock.MagicMock() + builder.recover_orphaned_artifact.return_value = [] + + start_next_batch_build(conf, module_build, builder) + + # Batch number should increase. + assert module_build.batch == 2 + + # Make sure we only have one message returned for the one reused component + assert len(events.scheduler.queue) == 1 + # The KojiBuildChange message in further_work should have build_new_state + # set to COMPLETE, but the current component build state in the DB should be set + # to BUILDING, so KojiBuildChange message handler handles the change + # properly. + event_info = events.scheduler.queue[0][3] + assert event_info == ('reuse_component: fake msg', 90276227, 1, 'perl-Tangerine', + '0.23', '1.module+0+d027b723', 3, + 'Reused component from previous module build') + component_build = models.ComponentBuild.from_component_event( + db_session, + task_id=event_info[1], + module_id=event_info[6], + ) + assert component_build.state == koji.BUILD_STATES["BUILDING"] + assert component_build.package == "perl-Tangerine" + assert component_build.reused_component_id is not None + # Make sure perl-List-Compare is set to the build state as well but not reused + assert plc_component.state == koji.BUILD_STATES["BUILDING"] + assert plc_component.reused_component_id is None + mock_sbc.assert_called_once() + + @patch("module_build_service.scheduler.batches.start_build_component") + @patch( + "module_build_service.config.Config.rebuild_strategy", + new_callable=mock.PropertyMock, + return_value="all", + ) + def test_start_next_batch_build_rebuild_strategy_all( + self, mock_rm, mock_sbc, default_buildroot_groups + ): + """ + Tests that start_next_batch_build can't reuse any components in the batch because the + rebuild method is set to "all". + """ + module_build = models.ModuleBuild.get_by_id(db_session, 3) + module_build.rebuild_strategy = "all" + module_build.batch = 1 + + builder = mock.MagicMock() + builder.recover_orphaned_artifact.return_value = [] + start_next_batch_build(conf, module_build, builder) + + # Batch number should increase. + assert module_build.batch == 2 + # No component reuse messages should be returned + assert len(events.scheduler.queue) == 0 + # Make sure that both components in the batch were submitted + assert len(mock_sbc.mock_calls) == 2 + + def test_start_build_component_failed_state(self, default_buildroot_groups): + """ + Tests whether exception occured while building sets the state to failed + """ + builder = mock.MagicMock() + builder.build.side_effect = Exception("Something have gone terribly wrong") + component = mock.MagicMock() + + start_build_component(db_session, builder, component) + + assert component.state == koji.BUILD_STATES["FAILED"] + + @patch("module_build_service.scheduler.batches.start_build_component") + @patch( + "module_build_service.config.Config.rebuild_strategy", + new_callable=mock.PropertyMock, + return_value="only-changed", + ) + def test_start_next_batch_build_rebuild_strategy_only_changed( + self, mock_rm, mock_sbc, default_buildroot_groups + ): + """ + Tests that start_next_batch_build reuses all unchanged components in the batch because the + rebuild method is set to "only-changed". This means that one component is reused in batch + 2, and even though the other component in batch 2 changed and was rebuilt, the component + in batch 3 can be reused. + """ + module_build = models.ModuleBuild.get_by_id(db_session, 3) + module_build.rebuild_strategy = "only-changed" + module_build.batch = 1 + # perl-List-Compare changed + plc_component = models.ComponentBuild.from_component_name( + db_session, "perl-List-Compare", 3) + plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd" + + builder = mock.MagicMock() + builder.recover_orphaned_artifact.return_value = [] + start_next_batch_build(conf, module_build, builder) + + # Batch number should increase + assert module_build.batch == 2 + + # Make sure we only have one message returned for the one reused component + assert len(events.scheduler.queue) == 1 + # The buildsys.build.state.change message in further_work should have + # build_new_state set to COMPLETE, but the current component build state + # in the DB should be set to BUILDING, so the build state change handler + # handles the change properly. + event_info = events.scheduler.queue[0][3] + assert event_info == ('reuse_component: fake msg', 90276227, 1, + 'perl-Tangerine', '0.23', '1.module+0+d027b723', 3, + 'Reused component from previous module build') + component_build = models.ComponentBuild.from_component_event( + db_session, + task_id=event_info[1], + module_id=event_info[6], + ) + assert component_build.state == koji.BUILD_STATES["BUILDING"] + assert component_build.package == "perl-Tangerine" + assert component_build.reused_component_id is not None + # Make sure perl-List-Compare is set to the build state as well but not reused + assert plc_component.state == koji.BUILD_STATES["BUILDING"] + assert plc_component.reused_component_id is None + mock_sbc.assert_called_once() + mock_sbc.reset_mock() + + # Complete the build + plc_component.state = koji.BUILD_STATES["COMPLETE"] + pt_component = models.ComponentBuild.from_component_name( + db_session, "perl-Tangerine", 3) + pt_component.state = koji.BUILD_STATES["COMPLETE"] + + events.scheduler.reset() + + # Start the next build batch + start_next_batch_build(conf, module_build, builder) + # Batch number should increase + assert module_build.batch == 3 + # Verify that tangerine was reused even though perl-Tangerine was rebuilt in the previous + # batch + event_info = events.scheduler.queue[0][3] + assert event_info == ('reuse_component: fake msg', 90276315, 1, 'tangerine', '0.22', + '3.module+0+d027b723', 3, + 'Reused component from previous module build') + component_build = models.ComponentBuild.from_component_event( + db_session, + task_id=event_info[1], + module_id=event_info[6], + ) + assert component_build.state == koji.BUILD_STATES["BUILDING"] + assert component_build.package == "tangerine" + assert component_build.reused_component_id is not None + mock_sbc.assert_not_called() + + @patch("module_build_service.scheduler.batches.start_build_component") + def test_start_next_batch_build_smart_scheduling( + self, mock_sbc, default_buildroot_groups + ): + """ + Tests that components with the longest build time will be scheduled first + """ + module_build = models.ModuleBuild.get_by_id(db_session, 3) + module_build.batch = 1 + pt_component = models.ComponentBuild.from_component_name( + db_session, "perl-Tangerine", 3) + pt_component.ref = "6ceea46add2366d8b8c5a623b2fb563b625bfabe" + plc_component = models.ComponentBuild.from_component_name( + db_session, "perl-List-Compare", 3) + plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd" + + # Components are by default built by component id. To find out that weight is respected, + # we have to set bigger weight to component with lower id. + pt_component.weight = 3 if pt_component.id < plc_component.id else 4 + plc_component.weight = 4 if pt_component.id < plc_component.id else 3 + + builder = mock.MagicMock() + builder.recover_orphaned_artifact.return_value = [] + start_next_batch_build(conf, module_build, builder) + + # Batch number should increase. + assert module_build.batch == 2 + + # Make sure we don't have any messages returned since no components should be reused + assert len(events.scheduler.queue) == 0 + # Make sure both components are set to the build state but not reused + assert pt_component.state == koji.BUILD_STATES["BUILDING"] + assert pt_component.reused_component_id is None + assert plc_component.state == koji.BUILD_STATES["BUILDING"] + assert plc_component.reused_component_id is None + + # Test the order of the scheduling + expected_calls = [ + mock.call(db_session, builder, plc_component), + mock.call(db_session, builder, pt_component) + ] + assert mock_sbc.mock_calls == expected_calls + + @patch("module_build_service.scheduler.batches.start_build_component") + def test_start_next_batch_continue(self, mock_sbc, default_buildroot_groups): + """ + Tests that start_next_batch_build does not start new batch when + there are unbuilt components in the current one. + """ + module_build = models.ModuleBuild.get_by_id(db_session, 3) + module_build.batch = 2 + + # The component was reused when the batch first started + building_component = module_build.current_batch()[0] + building_component.state = koji.BUILD_STATES["BUILDING"] + db_session.commit() + + builder = mock.MagicMock() + start_next_batch_build(conf, module_build, builder) + + # Batch number should not increase. + assert module_build.batch == 2 + # Make sure start build was called for the second component which wasn't reused + mock_sbc.assert_called_once() + # No further work should be returned + + assert len(events.scheduler.queue) == 0 + + def test_start_next_batch_build_repo_building(self, default_buildroot_groups): + """ + Test that start_next_batch_build does not start new batch when + builder.buildroot_ready() returns False. + """ + module_build = models.ModuleBuild.get_by_id(db_session, 3) + module_build.batch = 1 + db_session.commit() + + builder = mock.MagicMock() + builder.buildroot_ready.return_value = False + + # Batch number should not increase. + assert module_build.batch == 1 diff --git a/tests/test_scheduler/test_poller.py b/tests/test_scheduler/test_poller.py index 7ac5dc5f..e9851eef 100644 --- a/tests/test_scheduler/test_poller.py +++ b/tests/test_scheduler/test_poller.py @@ -35,7 +35,7 @@ class TestPoller: clean_database() @pytest.mark.parametrize("fresh", [True, False]) - @patch("module_build_service.utils.batches.start_build_component") + @patch("module_build_service.scheduler.batches.start_build_component") def test_process_paused_module_builds( self, start_build_component, create_builder, dbg, fresh ): @@ -81,7 +81,7 @@ class TestPoller: (koji.TASK_STATES["CLOSED"], True), (koji.TASK_STATES["OPEN"], False), )) - @patch("module_build_service.utils.batches.start_build_component") + @patch("module_build_service.scheduler.batches.start_build_component") def test_process_paused_module_builds_with_new_repo_task( self, start_build_component, create_builder, dbg, task_state, expect_start_build_component diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index e787fc3a..46be782e 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -25,12 +25,9 @@ from tests import ( make_module, read_staged_data, staged_data_filename) import mock -import koji import pytest import module_build_service.scheduler.handlers.components from module_build_service.db_session import db_session -from module_build_service.builder import GenericBuilder -from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder from module_build_service.scheduler import events from module_build_service import app, Modulemd @@ -1117,399 +1114,6 @@ class TestUtils: db_session, "foo", mmd_copy, {"branch": "otherbranch"}) -class DummyModuleBuilder(GenericBuilder): - """ - Dummy module builder - """ - - backend = "koji" - _build_id = 0 - - TAGGED_COMPONENTS = [] - - @module_build_service.utils.validate_koji_tag("tag_name") - def __init__(self, db_session, owner, module, config, tag_name, components): - self.db_session = db_session - self.module_str = module - self.tag_name = tag_name - self.config = config - - def buildroot_connect(self, groups): - pass - - def buildroot_prep(self): - pass - - def buildroot_resume(self): - pass - - def buildroot_ready(self, artifacts=None): - return True - - def buildroot_add_dependency(self, dependencies): - pass - - def buildroot_add_artifacts(self, artifacts, install=False): - DummyModuleBuilder.TAGGED_COMPONENTS += artifacts - - def buildroot_add_repos(self, dependencies): - pass - - def tag_artifacts(self, artifacts): - pass - - def recover_orphaned_artifact(self, component_build): - return [] - - @property - def module_build_tag(self): - return {"name": self.tag_name + "-build"} - - def build(self, artifact_name, source): - DummyModuleBuilder._build_id += 1 - state = koji.BUILD_STATES["COMPLETE"] - reason = "Submitted %s to Koji" % (artifact_name) - return DummyModuleBuilder._build_id, state, reason, None - - @staticmethod - def get_disttag_srpm(disttag, module_build): - # @FIXME - return KojiModuleBuilder.get_disttag_srpm(disttag, module_build) - - def cancel_build(self, task_id): - pass - - def list_tasks_for_components(self, component_builds=None, state="active"): - pass - - def repo_from_tag(self, config, tag_name, arch): - pass - - def finalize(self, succeeded=True): - pass - - -@pytest.mark.usefixtures("reuse_component_init_data") -@patch( - "module_build_service.builder.GenericBuilder.default_buildroot_groups", - return_value={"build": [], "srpm-build": []}, -) -class TestBatches: - def setup_method(self, test_method): - GenericBuilder.register_backend_class(DummyModuleBuilder) - events.scheduler.reset() - - def teardown_method(self, test_method): - # clean_database() - DummyModuleBuilder.TAGGED_COMPONENTS = [] - GenericBuilder.register_backend_class(KojiModuleBuilder) - events.scheduler.reset() - - def test_start_next_batch_build_reuse(self, default_buildroot_groups): - """ - Tests that start_next_batch_build: - 1) Increments module.batch. - 2) Can reuse all components in batch - 3) Returns proper further_work messages for reused components. - 4) Returns the fake Repo change message - 5) Handling the further_work messages lead to proper tagging of - reused components. - """ - module_build = models.ModuleBuild.get_by_id(db_session, 3) - module_build.batch = 1 - - builder = mock.MagicMock() - builder.module_build_tag = {"name": "module-fedora-27-build"} - module_build_service.utils.start_next_batch_build( - conf, module_build, builder) - - # Batch number should increase. - assert module_build.batch == 2 - - # buildsys.build.state.change messages in further_work should have - # build_new_state set to COMPLETE, but the current component build - # state should be set to BUILDING, so KojiBuildChange message handler - # handles the change properly. - for event in events.scheduler.queue: - event_info = event[3] - if event_info[0].startswith("reuse_component"): - assert event_info[2] == koji.BUILD_STATES["COMPLETE"] - component_build = models.ComponentBuild.from_component_event( - db_session, - task_id=event_info[1], - module_id=event_info[6]) - assert component_build.state == koji.BUILD_STATES["BUILDING"] - - # When we handle these KojiBuildChange messages, MBS should tag all - # the components just once. - events.scheduler.run() - - # Check that packages have been tagged just once. - assert len(DummyModuleBuilder.TAGGED_COMPONENTS) == 2 - - @patch("module_build_service.utils.batches.start_build_component") - def test_start_next_batch_build_reuse_some( - self, mock_sbc, default_buildroot_groups - ): - """ - Tests that start_next_batch_build: - 1) Increments module.batch. - 2) Can reuse all components in the batch that it can. - 3) Returns proper further_work messages for reused components. - 4) Builds the remaining components - 5) Handling the further_work messages lead to proper tagging of - reused components. - """ - module_build = models.ModuleBuild.get_by_id(db_session, 3) - module_build.batch = 1 - - plc_component = models.ComponentBuild.from_component_name( - db_session, "perl-List-Compare", 3) - plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd" - - builder = mock.MagicMock() - builder.recover_orphaned_artifact.return_value = [] - - module_build_service.utils.start_next_batch_build( - conf, module_build, builder) - - # Batch number should increase. - assert module_build.batch == 2 - - # Make sure we only have one message returned for the one reused component - assert len(events.scheduler.queue) == 1 - # The KojiBuildChange message in further_work should have build_new_state - # set to COMPLETE, but the current component build state in the DB should be set - # to BUILDING, so KojiBuildChange message handler handles the change - # properly. - event_info = events.scheduler.queue[0][3] - assert event_info == ('reuse_component: fake msg', 90276227, 1, 'perl-Tangerine', - '0.23', '1.module+0+d027b723', 3, - 'Reused component from previous module build') - component_build = models.ComponentBuild.from_component_event( - db_session, - task_id=event_info[1], - module_id=event_info[6], - ) - assert component_build.state == koji.BUILD_STATES["BUILDING"] - assert component_build.package == "perl-Tangerine" - assert component_build.reused_component_id is not None - # Make sure perl-List-Compare is set to the build state as well but not reused - assert plc_component.state == koji.BUILD_STATES["BUILDING"] - assert plc_component.reused_component_id is None - mock_sbc.assert_called_once() - - @patch("module_build_service.utils.batches.start_build_component") - @patch( - "module_build_service.config.Config.rebuild_strategy", - new_callable=mock.PropertyMock, - return_value="all", - ) - def test_start_next_batch_build_rebuild_strategy_all( - self, mock_rm, mock_sbc, default_buildroot_groups - ): - """ - Tests that start_next_batch_build can't reuse any components in the batch because the - rebuild method is set to "all". - """ - module_build = models.ModuleBuild.get_by_id(db_session, 3) - module_build.rebuild_strategy = "all" - module_build.batch = 1 - - builder = mock.MagicMock() - builder.recover_orphaned_artifact.return_value = [] - module_build_service.utils.start_next_batch_build( - conf, module_build, builder) - - # Batch number should increase. - assert module_build.batch == 2 - # No component reuse messages should be returned - assert len(events.scheduler.queue) == 0 - # Make sure that both components in the batch were submitted - assert len(mock_sbc.mock_calls) == 2 - - def test_start_build_component_failed_state(self, default_buildroot_groups): - """ - Tests whether exception occured while building sets the state to failed - """ - builder = mock.MagicMock() - builder.build.side_effect = Exception("Something have gone terribly wrong") - component = mock.MagicMock() - - module_build_service.utils.batches.start_build_component(db_session, builder, component) - - assert component.state == koji.BUILD_STATES["FAILED"] - - @patch("module_build_service.utils.batches.start_build_component") - @patch( - "module_build_service.config.Config.rebuild_strategy", - new_callable=mock.PropertyMock, - return_value="only-changed", - ) - def test_start_next_batch_build_rebuild_strategy_only_changed( - self, mock_rm, mock_sbc, default_buildroot_groups - ): - """ - Tests that start_next_batch_build reuses all unchanged components in the batch because the - rebuild method is set to "only-changed". This means that one component is reused in batch - 2, and even though the other component in batch 2 changed and was rebuilt, the component - in batch 3 can be reused. - """ - module_build = models.ModuleBuild.get_by_id(db_session, 3) - module_build.rebuild_strategy = "only-changed" - module_build.batch = 1 - # perl-List-Compare changed - plc_component = models.ComponentBuild.from_component_name( - db_session, "perl-List-Compare", 3) - plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd" - - builder = mock.MagicMock() - builder.recover_orphaned_artifact.return_value = [] - module_build_service.utils.start_next_batch_build( - conf, module_build, builder) - - # Batch number should increase - assert module_build.batch == 2 - - # Make sure we only have one message returned for the one reused component - assert len(events.scheduler.queue) == 1 - # The buildsys.build.state.change message in further_work should have - # build_new_state set to COMPLETE, but the current component build state - # in the DB should be set to BUILDING, so the build state change handler - # handles the change properly. - event_info = events.scheduler.queue[0][3] - assert event_info == ('reuse_component: fake msg', 90276227, 1, - 'perl-Tangerine', '0.23', '1.module+0+d027b723', 3, - 'Reused component from previous module build') - component_build = models.ComponentBuild.from_component_event( - db_session, - task_id=event_info[1], - module_id=event_info[6], - ) - assert component_build.state == koji.BUILD_STATES["BUILDING"] - assert component_build.package == "perl-Tangerine" - assert component_build.reused_component_id is not None - # Make sure perl-List-Compare is set to the build state as well but not reused - assert plc_component.state == koji.BUILD_STATES["BUILDING"] - assert plc_component.reused_component_id is None - mock_sbc.assert_called_once() - mock_sbc.reset_mock() - - # Complete the build - plc_component.state = koji.BUILD_STATES["COMPLETE"] - pt_component = models.ComponentBuild.from_component_name( - db_session, "perl-Tangerine", 3) - pt_component.state = koji.BUILD_STATES["COMPLETE"] - - events.scheduler.reset() - - # Start the next build batch - module_build_service.utils.start_next_batch_build( - conf, module_build, builder) - # Batch number should increase - assert module_build.batch == 3 - # Verify that tangerine was reused even though perl-Tangerine was rebuilt in the previous - # batch - event_info = events.scheduler.queue[0][3] - assert event_info == ('reuse_component: fake msg', 90276315, 1, 'tangerine', '0.22', - '3.module+0+d027b723', 3, - 'Reused component from previous module build') - component_build = models.ComponentBuild.from_component_event( - db_session, - task_id=event_info[1], - module_id=event_info[6], - ) - assert component_build.state == koji.BUILD_STATES["BUILDING"] - assert component_build.package == "tangerine" - assert component_build.reused_component_id is not None - mock_sbc.assert_not_called() - - @patch("module_build_service.utils.batches.start_build_component") - def test_start_next_batch_build_smart_scheduling( - self, mock_sbc, default_buildroot_groups - ): - """ - Tests that components with the longest build time will be scheduled first - """ - module_build = models.ModuleBuild.get_by_id(db_session, 3) - module_build.batch = 1 - pt_component = models.ComponentBuild.from_component_name( - db_session, "perl-Tangerine", 3) - pt_component.ref = "6ceea46add2366d8b8c5a623b2fb563b625bfabe" - plc_component = models.ComponentBuild.from_component_name( - db_session, "perl-List-Compare", 3) - plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd" - - # Components are by default built by component id. To find out that weight is respected, - # we have to set bigger weight to component with lower id. - pt_component.weight = 3 if pt_component.id < plc_component.id else 4 - plc_component.weight = 4 if pt_component.id < plc_component.id else 3 - - builder = mock.MagicMock() - builder.recover_orphaned_artifact.return_value = [] - module_build_service.utils.start_next_batch_build( - conf, module_build, builder) - - # Batch number should increase. - assert module_build.batch == 2 - - # Make sure we don't have any messages returned since no components should be reused - assert len(events.scheduler.queue) == 0 - # Make sure both components are set to the build state but not reused - assert pt_component.state == koji.BUILD_STATES["BUILDING"] - assert pt_component.reused_component_id is None - assert plc_component.state == koji.BUILD_STATES["BUILDING"] - assert plc_component.reused_component_id is None - - # Test the order of the scheduling - expected_calls = [ - mock.call(db_session, builder, plc_component), - mock.call(db_session, builder, pt_component) - ] - assert mock_sbc.mock_calls == expected_calls - - @patch("module_build_service.utils.batches.start_build_component") - def test_start_next_batch_continue(self, mock_sbc, default_buildroot_groups): - """ - Tests that start_next_batch_build does not start new batch when - there are unbuilt components in the current one. - """ - module_build = models.ModuleBuild.get_by_id(db_session, 3) - module_build.batch = 2 - - # The component was reused when the batch first started - building_component = module_build.current_batch()[0] - building_component.state = koji.BUILD_STATES["BUILDING"] - db_session.commit() - - builder = mock.MagicMock() - module_build_service.utils.start_next_batch_build( - conf, module_build, builder) - - # Batch number should not increase. - assert module_build.batch == 2 - # Make sure start build was called for the second component which wasn't reused - mock_sbc.assert_called_once() - # No further work should be returned - - assert len(events.scheduler.queue) == 0 - - def test_start_next_batch_build_repo_building(self, default_buildroot_groups): - """ - Test that start_next_batch_build does not start new batch when - builder.buildroot_ready() returns False. - """ - module_build = models.ModuleBuild.get_by_id(db_session, 3) - module_build.batch = 1 - db_session.commit() - - builder = mock.MagicMock() - builder.buildroot_ready.return_value = False - - # Batch number should not increase. - assert module_build.batch == 1 - - @patch( "module_build_service.config.Config.mock_resultsdir", new_callable=mock.PropertyMock,