modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-02 02:11:19 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Move utils/batches.py to scheduler/batches.py

Browse Source
This commit is contained in:
mprahl
2020-01-02 14:22:33 -05:00
parent 938a3b9557
commit 81ab984b1f
10 changed files with 415 additions and 409 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/HOW_MBS_BUILDS_MODULES.rst
View File

@@ -156,7 +156,7 @@ the ``scheduler.handlers.repos.done(...)`` method is called.
This verifies that all the packages from the current batch (just module-build-macros for now)
really appear in the generated repository and if so, it starts building the next batch by calling
``utils.batches.start_next_batch_build(...)``.
``module_build_service.scheduler.batches.start_next_batch_build(...)``.
Building the next batch

4
module_build_service/utils/batches.py → module_build_service/scheduler/batches.py
View File

@@ -1,12 +1,12 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
import threading
import concurrent.futures
import threading
from module_build_service import conf, log, models
from module_build_service.db_session import db_session
from module_build_service.scheduler import events
from .reuse import get_reusable_components, reuse_component
from module_build_service.utils.reuse import get_reusable_components, reuse_component
def at_concurrent_component_threshold(config):

2
module_build_service/scheduler/handlers/components.py
View File

@@ -11,7 +11,7 @@ from module_build_service.common.koji import get_session
from module_build_service.utils.general import mmd_to_str
from module_build_service.db_session import db_session
from module_build_service.scheduler import events
from module_build_service.utils.batches import continue_batch_build
from module_build_service.scheduler.batches import continue_batch_build
logging.basicConfig(level=logging.DEBUG)

2
module_build_service/scheduler/handlers/repos.py
View File

@@ -6,7 +6,7 @@ import logging
from datetime import datetime
from module_build_service import celery_app, conf, models, log
from module_build_service.builder import GenericBuilder
from module_build_service.utils import start_next_batch_build
from module_build_service.scheduler.batches import start_next_batch_build
from module_build_service.db_session import db_session
from module_build_service.scheduler import events

11
module_build_service/scheduler/producer.py
View File

@@ -14,6 +14,10 @@ from module_build_service.builder import GenericBuilder
from module_build_service.common.koji import get_session
from module_build_service.utils.greenwave import greenwave
from module_build_service.db_session import db_session
from module_build_service.scheduler.batches import (
at_concurrent_component_threshold,
start_next_batch_build,
)
from module_build_service.scheduler.consumer import ON_MODULE_CHANGE_HANDLERS
from module_build_service.scheduler.handlers.components import build_task_finalize
from module_build_service.scheduler.handlers.tags import tagged
@@ -168,7 +172,7 @@ def fail_lost_builds():
@celery_app.task
def process_paused_module_builds():
log.info("Looking for paused module builds in the build state")
if module_build_service.utils.at_concurrent_component_threshold(conf):
if at_concurrent_component_threshold(conf):
log.debug(
"Will not attempt to start paused module builds due to "
"the concurrent build threshold being met"
@@ -199,11 +203,10 @@ def process_paused_module_builds():
if has_missed_new_repo_message(module_build, builder.koji_session):
log.info(" Processing the paused module build %r", module_build)
module_build_service.utils.start_next_batch_build(
conf, module_build, builder)
start_next_batch_build(conf, module_build, builder)
# Check if we have met the threshold.
if module_build_service.utils.at_concurrent_component_threshold(conf):
if at_concurrent_component_threshold(conf):
break

1
module_build_service/utils/__init__.py
View File

@@ -5,5 +5,4 @@ from module_build_service.utils.mse import * # noqa
from module_build_service.utils.views import * # noqa
from module_build_service.utils.reuse import * # noqa
from module_build_service.utils.submit import * # noqa
from module_build_service.utils.batches import * # noqa
from module_build_service.utils.ursine import * # noqa

2
tests/test_build/test_build.py
View File

@@ -1789,7 +1789,7 @@ class TestBuild(BaseTestBuild):
return result
with patch(
"module_build_service.utils.batches.at_concurrent_component_threshold"
"module_build_service.scheduler.batches.at_concurrent_component_threshold"
) as mock_acct:
# Once we get to batch 2, then simulate the concurrent threshold being met
def _at_concurrent_component_threshold(config):

400
tests/test_scheduler/test_batches.py Normal file
View File

@@ -0,0 +1,400 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
from mock import patch
from module_build_service import models, conf
import mock
import koji
import pytest
from module_build_service.db_session import db_session
from module_build_service.builder import GenericBuilder
from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder
from module_build_service.scheduler import events
from module_build_service.scheduler.batches import start_build_component, start_next_batch_build
from module_build_service.utils import validate_koji_tag
class DummyModuleBuilder(GenericBuilder):
"""
Dummy module builder
"""
backend = "koji"
_build_id = 0
TAGGED_COMPONENTS = []
@validate_koji_tag("tag_name")
def __init__(self, db_session, owner, module, config, tag_name, components):
self.db_session = db_session
self.module_str = module
self.tag_name = tag_name
self.config = config
def buildroot_connect(self, groups):
pass
def buildroot_prep(self):
pass
def buildroot_resume(self):
pass
def buildroot_ready(self, artifacts=None):
return True
def buildroot_add_dependency(self, dependencies):
pass
def buildroot_add_artifacts(self, artifacts, install=False):
DummyModuleBuilder.TAGGED_COMPONENTS += artifacts
def buildroot_add_repos(self, dependencies):
pass
def tag_artifacts(self, artifacts):
pass
def recover_orphaned_artifact(self, component_build):
return []
@property
def module_build_tag(self):
return {"name": self.tag_name + "-build"}
def build(self, artifact_name, source):
DummyModuleBuilder._build_id += 1
state = koji.BUILD_STATES["COMPLETE"]
reason = "Submitted %s to Koji" % (artifact_name)
return DummyModuleBuilder._build_id, state, reason, None
@staticmethod
def get_disttag_srpm(disttag, module_build):
# @FIXME
return KojiModuleBuilder.get_disttag_srpm(disttag, module_build)
def cancel_build(self, task_id):
pass
def list_tasks_for_components(self, component_builds=None, state="active"):
pass
def repo_from_tag(self, config, tag_name, arch):
pass
def finalize(self, succeeded=True):
pass
@pytest.mark.usefixtures("reuse_component_init_data")
@patch(
"module_build_service.builder.GenericBuilder.default_buildroot_groups",
return_value={"build": [], "srpm-build": []},
)
class TestBatches:
def setup_method(self, test_method):
GenericBuilder.register_backend_class(DummyModuleBuilder)
events.scheduler.reset()
def teardown_method(self, test_method):
# clean_database()
DummyModuleBuilder.TAGGED_COMPONENTS = []
GenericBuilder.register_backend_class(KojiModuleBuilder)
events.scheduler.reset()
def test_start_next_batch_build_reuse(self, default_buildroot_groups):
"""
Tests that start_next_batch_build:
1) Increments module.batch.
2) Can reuse all components in batch
3) Returns proper further_work messages for reused components.
4) Returns the fake Repo change message
5) Handling the further_work messages lead to proper tagging of
reused components.
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.batch = 1
builder = mock.MagicMock()
builder.module_build_tag = {"name": "module-fedora-27-build"}
start_next_batch_build(
conf, module_build, builder)
# Batch number should increase.
assert module_build.batch == 2
# buildsys.build.state.change messages in further_work should have
# build_new_state set to COMPLETE, but the current component build
# state should be set to BUILDING, so KojiBuildChange message handler
# handles the change properly.
for event in events.scheduler.queue:
event_info = event[3]
if event_info[0].startswith("reuse_component"):
assert event_info[2] == koji.BUILD_STATES["COMPLETE"]
component_build = models.ComponentBuild.from_component_event(
db_session,
task_id=event_info[1],
module_id=event_info[6])
assert component_build.state == koji.BUILD_STATES["BUILDING"]
# When we handle these KojiBuildChange messages, MBS should tag all
# the components just once.
events.scheduler.run()
# Check that packages have been tagged just once.
assert len(DummyModuleBuilder.TAGGED_COMPONENTS) == 2
@patch("module_build_service.scheduler.batches.start_build_component")
def test_start_next_batch_build_reuse_some(
self, mock_sbc, default_buildroot_groups
):
"""
Tests that start_next_batch_build:
1) Increments module.batch.
2) Can reuse all components in the batch that it can.
3) Returns proper further_work messages for reused components.
4) Builds the remaining components
5) Handling the further_work messages lead to proper tagging of
reused components.
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.batch = 1
plc_component = models.ComponentBuild.from_component_name(
db_session, "perl-List-Compare", 3)
plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd"
builder = mock.MagicMock()
builder.recover_orphaned_artifact.return_value = []
start_next_batch_build(conf, module_build, builder)
# Batch number should increase.
assert module_build.batch == 2
# Make sure we only have one message returned for the one reused component
assert len(events.scheduler.queue) == 1
# The KojiBuildChange message in further_work should have build_new_state
# set to COMPLETE, but the current component build state in the DB should be set
# to BUILDING, so KojiBuildChange message handler handles the change
# properly.
event_info = events.scheduler.queue[0][3]
assert event_info == ('reuse_component: fake msg', 90276227, 1, 'perl-Tangerine',
'0.23', '1.module+0+d027b723', 3,
'Reused component from previous module build')
component_build = models.ComponentBuild.from_component_event(
db_session,
task_id=event_info[1],
module_id=event_info[6],
)
assert component_build.state == koji.BUILD_STATES["BUILDING"]
assert component_build.package == "perl-Tangerine"
assert component_build.reused_component_id is not None
# Make sure perl-List-Compare is set to the build state as well but not reused
assert plc_component.state == koji.BUILD_STATES["BUILDING"]
assert plc_component.reused_component_id is None
mock_sbc.assert_called_once()
@patch("module_build_service.scheduler.batches.start_build_component")
@patch(
"module_build_service.config.Config.rebuild_strategy",
new_callable=mock.PropertyMock,
return_value="all",
)
def test_start_next_batch_build_rebuild_strategy_all(
self, mock_rm, mock_sbc, default_buildroot_groups
):
"""
Tests that start_next_batch_build can't reuse any components in the batch because the
rebuild method is set to "all".
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.rebuild_strategy = "all"
module_build.batch = 1
builder = mock.MagicMock()
builder.recover_orphaned_artifact.return_value = []
start_next_batch_build(conf, module_build, builder)
# Batch number should increase.
assert module_build.batch == 2
# No component reuse messages should be returned
assert len(events.scheduler.queue) == 0
# Make sure that both components in the batch were submitted
assert len(mock_sbc.mock_calls) == 2
def test_start_build_component_failed_state(self, default_buildroot_groups):
"""
Tests whether exception occured while building sets the state to failed
"""
builder = mock.MagicMock()
builder.build.side_effect = Exception("Something have gone terribly wrong")
component = mock.MagicMock()
start_build_component(db_session, builder, component)
assert component.state == koji.BUILD_STATES["FAILED"]
@patch("module_build_service.scheduler.batches.start_build_component")
@patch(
"module_build_service.config.Config.rebuild_strategy",
new_callable=mock.PropertyMock,
return_value="only-changed",
)
def test_start_next_batch_build_rebuild_strategy_only_changed(
self, mock_rm, mock_sbc, default_buildroot_groups
):
"""
Tests that start_next_batch_build reuses all unchanged components in the batch because the
rebuild method is set to "only-changed". This means that one component is reused in batch
2, and even though the other component in batch 2 changed and was rebuilt, the component
in batch 3 can be reused.
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.rebuild_strategy = "only-changed"
module_build.batch = 1
# perl-List-Compare changed
plc_component = models.ComponentBuild.from_component_name(
db_session, "perl-List-Compare", 3)
plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd"
builder = mock.MagicMock()
builder.recover_orphaned_artifact.return_value = []
start_next_batch_build(conf, module_build, builder)
# Batch number should increase
assert module_build.batch == 2
# Make sure we only have one message returned for the one reused component
assert len(events.scheduler.queue) == 1
# The buildsys.build.state.change message in further_work should have
# build_new_state set to COMPLETE, but the current component build state
# in the DB should be set to BUILDING, so the build state change handler
# handles the change properly.
event_info = events.scheduler.queue[0][3]
assert event_info == ('reuse_component: fake msg', 90276227, 1,
'perl-Tangerine', '0.23', '1.module+0+d027b723', 3,
'Reused component from previous module build')
component_build = models.ComponentBuild.from_component_event(
db_session,
task_id=event_info[1],
module_id=event_info[6],
)
assert component_build.state == koji.BUILD_STATES["BUILDING"]
assert component_build.package == "perl-Tangerine"
assert component_build.reused_component_id is not None
# Make sure perl-List-Compare is set to the build state as well but not reused
assert plc_component.state == koji.BUILD_STATES["BUILDING"]
assert plc_component.reused_component_id is None
mock_sbc.assert_called_once()
mock_sbc.reset_mock()
# Complete the build
plc_component.state = koji.BUILD_STATES["COMPLETE"]
pt_component = models.ComponentBuild.from_component_name(
db_session, "perl-Tangerine", 3)
pt_component.state = koji.BUILD_STATES["COMPLETE"]
events.scheduler.reset()
# Start the next build batch
start_next_batch_build(conf, module_build, builder)
# Batch number should increase
assert module_build.batch == 3
# Verify that tangerine was reused even though perl-Tangerine was rebuilt in the previous
# batch
event_info = events.scheduler.queue[0][3]
assert event_info == ('reuse_component: fake msg', 90276315, 1, 'tangerine', '0.22',
'3.module+0+d027b723', 3,
'Reused component from previous module build')
component_build = models.ComponentBuild.from_component_event(
db_session,
task_id=event_info[1],
module_id=event_info[6],
)
assert component_build.state == koji.BUILD_STATES["BUILDING"]
assert component_build.package == "tangerine"
assert component_build.reused_component_id is not None
mock_sbc.assert_not_called()
@patch("module_build_service.scheduler.batches.start_build_component")
def test_start_next_batch_build_smart_scheduling(
self, mock_sbc, default_buildroot_groups
):
"""
Tests that components with the longest build time will be scheduled first
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.batch = 1
pt_component = models.ComponentBuild.from_component_name(
db_session, "perl-Tangerine", 3)
pt_component.ref = "6ceea46add2366d8b8c5a623b2fb563b625bfabe"
plc_component = models.ComponentBuild.from_component_name(
db_session, "perl-List-Compare", 3)
plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd"
# Components are by default built by component id. To find out that weight is respected,
# we have to set bigger weight to component with lower id.
pt_component.weight = 3 if pt_component.id < plc_component.id else 4
plc_component.weight = 4 if pt_component.id < plc_component.id else 3
builder = mock.MagicMock()
builder.recover_orphaned_artifact.return_value = []
start_next_batch_build(conf, module_build, builder)
# Batch number should increase.
assert module_build.batch == 2
# Make sure we don't have any messages returned since no components should be reused
assert len(events.scheduler.queue) == 0
# Make sure both components are set to the build state but not reused
assert pt_component.state == koji.BUILD_STATES["BUILDING"]
assert pt_component.reused_component_id is None
assert plc_component.state == koji.BUILD_STATES["BUILDING"]
assert plc_component.reused_component_id is None
# Test the order of the scheduling
expected_calls = [
mock.call(db_session, builder, plc_component),
mock.call(db_session, builder, pt_component)
]
assert mock_sbc.mock_calls == expected_calls
@patch("module_build_service.scheduler.batches.start_build_component")
def test_start_next_batch_continue(self, mock_sbc, default_buildroot_groups):
"""
Tests that start_next_batch_build does not start new batch when
there are unbuilt components in the current one.
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.batch = 2
# The component was reused when the batch first started
building_component = module_build.current_batch()[0]
building_component.state = koji.BUILD_STATES["BUILDING"]
db_session.commit()
builder = mock.MagicMock()
start_next_batch_build(conf, module_build, builder)
# Batch number should not increase.
assert module_build.batch == 2
# Make sure start build was called for the second component which wasn't reused
mock_sbc.assert_called_once()
# No further work should be returned
assert len(events.scheduler.queue) == 0
def test_start_next_batch_build_repo_building(self, default_buildroot_groups):
"""
Test that start_next_batch_build does not start new batch when
builder.buildroot_ready() returns False.
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.batch = 1
db_session.commit()
builder = mock.MagicMock()
builder.buildroot_ready.return_value = False
# Batch number should not increase.
assert module_build.batch == 1

4
tests/test_scheduler/test_poller.py
View File

@@ -35,7 +35,7 @@ class TestPoller:
clean_database()
@pytest.mark.parametrize("fresh", [True, False])
@patch("module_build_service.utils.batches.start_build_component")
@patch("module_build_service.scheduler.batches.start_build_component")
def test_process_paused_module_builds(
self, start_build_component, create_builder, dbg, fresh
):
@@ -81,7 +81,7 @@ class TestPoller:
(koji.TASK_STATES["CLOSED"], True),
(koji.TASK_STATES["OPEN"], False),
))
@patch("module_build_service.utils.batches.start_build_component")
@patch("module_build_service.scheduler.batches.start_build_component")
def test_process_paused_module_builds_with_new_repo_task(
self, start_build_component, create_builder, dbg, task_state,
expect_start_build_component

396
tests/test_utils/test_utils.py
View File

@@ -25,12 +25,9 @@ from tests import (
make_module,
read_staged_data, staged_data_filename)
import mock
import koji
import pytest
import module_build_service.scheduler.handlers.components
from module_build_service.db_session import db_session
from module_build_service.builder import GenericBuilder
from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder
from module_build_service.scheduler import events
from module_build_service import app, Modulemd
@@ -1117,399 +1114,6 @@ class TestUtils:
db_session, "foo", mmd_copy, {"branch": "otherbranch"})
class DummyModuleBuilder(GenericBuilder):
"""
Dummy module builder
"""
backend = "koji"
_build_id = 0
TAGGED_COMPONENTS = []
@module_build_service.utils.validate_koji_tag("tag_name")
def __init__(self, db_session, owner, module, config, tag_name, components):
self.db_session = db_session
self.module_str = module
self.tag_name = tag_name
self.config = config
def buildroot_connect(self, groups):
pass
def buildroot_prep(self):
pass
def buildroot_resume(self):
pass
def buildroot_ready(self, artifacts=None):
return True
def buildroot_add_dependency(self, dependencies):
pass
def buildroot_add_artifacts(self, artifacts, install=False):
DummyModuleBuilder.TAGGED_COMPONENTS += artifacts
def buildroot_add_repos(self, dependencies):
pass
def tag_artifacts(self, artifacts):
pass
def recover_orphaned_artifact(self, component_build):
return []
@property
def module_build_tag(self):
return {"name": self.tag_name + "-build"}
def build(self, artifact_name, source):
DummyModuleBuilder._build_id += 1
state = koji.BUILD_STATES["COMPLETE"]
reason = "Submitted %s to Koji" % (artifact_name)
return DummyModuleBuilder._build_id, state, reason, None
@staticmethod
def get_disttag_srpm(disttag, module_build):
# @FIXME
return KojiModuleBuilder.get_disttag_srpm(disttag, module_build)
def cancel_build(self, task_id):
pass
def list_tasks_for_components(self, component_builds=None, state="active"):
pass
def repo_from_tag(self, config, tag_name, arch):
pass
def finalize(self, succeeded=True):
pass
@pytest.mark.usefixtures("reuse_component_init_data")
@patch(
"module_build_service.builder.GenericBuilder.default_buildroot_groups",
return_value={"build": [], "srpm-build": []},
)
class TestBatches:
def setup_method(self, test_method):
GenericBuilder.register_backend_class(DummyModuleBuilder)
events.scheduler.reset()
def teardown_method(self, test_method):
# clean_database()
DummyModuleBuilder.TAGGED_COMPONENTS = []
GenericBuilder.register_backend_class(KojiModuleBuilder)
events.scheduler.reset()
def test_start_next_batch_build_reuse(self, default_buildroot_groups):
"""
Tests that start_next_batch_build:
1) Increments module.batch.
2) Can reuse all components in batch
3) Returns proper further_work messages for reused components.
4) Returns the fake Repo change message
5) Handling the further_work messages lead to proper tagging of
reused components.
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.batch = 1
builder = mock.MagicMock()
builder.module_build_tag = {"name": "module-fedora-27-build"}
module_build_service.utils.start_next_batch_build(
conf, module_build, builder)
# Batch number should increase.
assert module_build.batch == 2
# buildsys.build.state.change messages in further_work should have
# build_new_state set to COMPLETE, but the current component build
# state should be set to BUILDING, so KojiBuildChange message handler
# handles the change properly.
for event in events.scheduler.queue:
event_info = event[3]
if event_info[0].startswith("reuse_component"):
assert event_info[2] == koji.BUILD_STATES["COMPLETE"]
component_build = models.ComponentBuild.from_component_event(
db_session,
task_id=event_info[1],
module_id=event_info[6])
assert component_build.state == koji.BUILD_STATES["BUILDING"]
# When we handle these KojiBuildChange messages, MBS should tag all
# the components just once.
events.scheduler.run()
# Check that packages have been tagged just once.
assert len(DummyModuleBuilder.TAGGED_COMPONENTS) == 2
@patch("module_build_service.utils.batches.start_build_component")
def test_start_next_batch_build_reuse_some(
self, mock_sbc, default_buildroot_groups
):
"""
Tests that start_next_batch_build:
1) Increments module.batch.
2) Can reuse all components in the batch that it can.
3) Returns proper further_work messages for reused components.
4) Builds the remaining components
5) Handling the further_work messages lead to proper tagging of
reused components.
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.batch = 1
plc_component = models.ComponentBuild.from_component_name(
db_session, "perl-List-Compare", 3)
plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd"
builder = mock.MagicMock()
builder.recover_orphaned_artifact.return_value = []
module_build_service.utils.start_next_batch_build(
conf, module_build, builder)
# Batch number should increase.
assert module_build.batch == 2
# Make sure we only have one message returned for the one reused component
assert len(events.scheduler.queue) == 1
# The KojiBuildChange message in further_work should have build_new_state
# set to COMPLETE, but the current component build state in the DB should be set
# to BUILDING, so KojiBuildChange message handler handles the change
# properly.
event_info = events.scheduler.queue[0][3]
assert event_info == ('reuse_component: fake msg', 90276227, 1, 'perl-Tangerine',
'0.23', '1.module+0+d027b723', 3,
'Reused component from previous module build')
component_build = models.ComponentBuild.from_component_event(
db_session,
task_id=event_info[1],
module_id=event_info[6],
)
assert component_build.state == koji.BUILD_STATES["BUILDING"]
assert component_build.package == "perl-Tangerine"
assert component_build.reused_component_id is not None
# Make sure perl-List-Compare is set to the build state as well but not reused
assert plc_component.state == koji.BUILD_STATES["BUILDING"]
assert plc_component.reused_component_id is None
mock_sbc.assert_called_once()
@patch("module_build_service.utils.batches.start_build_component")
@patch(
"module_build_service.config.Config.rebuild_strategy",
new_callable=mock.PropertyMock,
return_value="all",
)
def test_start_next_batch_build_rebuild_strategy_all(
self, mock_rm, mock_sbc, default_buildroot_groups
):
"""
Tests that start_next_batch_build can't reuse any components in the batch because the
rebuild method is set to "all".
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.rebuild_strategy = "all"
module_build.batch = 1
builder = mock.MagicMock()
builder.recover_orphaned_artifact.return_value = []
module_build_service.utils.start_next_batch_build(
conf, module_build, builder)
# Batch number should increase.
assert module_build.batch == 2
# No component reuse messages should be returned
assert len(events.scheduler.queue) == 0
# Make sure that both components in the batch were submitted
assert len(mock_sbc.mock_calls) == 2
def test_start_build_component_failed_state(self, default_buildroot_groups):
"""
Tests whether exception occured while building sets the state to failed
"""
builder = mock.MagicMock()
builder.build.side_effect = Exception("Something have gone terribly wrong")
component = mock.MagicMock()
module_build_service.utils.batches.start_build_component(db_session, builder, component)
assert component.state == koji.BUILD_STATES["FAILED"]
@patch("module_build_service.utils.batches.start_build_component")
@patch(
"module_build_service.config.Config.rebuild_strategy",
new_callable=mock.PropertyMock,
return_value="only-changed",
)
def test_start_next_batch_build_rebuild_strategy_only_changed(
self, mock_rm, mock_sbc, default_buildroot_groups
):
"""
Tests that start_next_batch_build reuses all unchanged components in the batch because the
rebuild method is set to "only-changed". This means that one component is reused in batch
2, and even though the other component in batch 2 changed and was rebuilt, the component
in batch 3 can be reused.
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.rebuild_strategy = "only-changed"
module_build.batch = 1
# perl-List-Compare changed
plc_component = models.ComponentBuild.from_component_name(
db_session, "perl-List-Compare", 3)
plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd"
builder = mock.MagicMock()
builder.recover_orphaned_artifact.return_value = []
module_build_service.utils.start_next_batch_build(
conf, module_build, builder)
# Batch number should increase
assert module_build.batch == 2
# Make sure we only have one message returned for the one reused component
assert len(events.scheduler.queue) == 1
# The buildsys.build.state.change message in further_work should have
# build_new_state set to COMPLETE, but the current component build state
# in the DB should be set to BUILDING, so the build state change handler
# handles the change properly.
event_info = events.scheduler.queue[0][3]
assert event_info == ('reuse_component: fake msg', 90276227, 1,
'perl-Tangerine', '0.23', '1.module+0+d027b723', 3,
'Reused component from previous module build')
component_build = models.ComponentBuild.from_component_event(
db_session,
task_id=event_info[1],
module_id=event_info[6],
)
assert component_build.state == koji.BUILD_STATES["BUILDING"]
assert component_build.package == "perl-Tangerine"
assert component_build.reused_component_id is not None
# Make sure perl-List-Compare is set to the build state as well but not reused
assert plc_component.state == koji.BUILD_STATES["BUILDING"]
assert plc_component.reused_component_id is None
mock_sbc.assert_called_once()
mock_sbc.reset_mock()
# Complete the build
plc_component.state = koji.BUILD_STATES["COMPLETE"]
pt_component = models.ComponentBuild.from_component_name(
db_session, "perl-Tangerine", 3)
pt_component.state = koji.BUILD_STATES["COMPLETE"]
events.scheduler.reset()
# Start the next build batch
module_build_service.utils.start_next_batch_build(
conf, module_build, builder)
# Batch number should increase
assert module_build.batch == 3
# Verify that tangerine was reused even though perl-Tangerine was rebuilt in the previous
# batch
event_info = events.scheduler.queue[0][3]
assert event_info == ('reuse_component: fake msg', 90276315, 1, 'tangerine', '0.22',
'3.module+0+d027b723', 3,
'Reused component from previous module build')
component_build = models.ComponentBuild.from_component_event(
db_session,
task_id=event_info[1],
module_id=event_info[6],
)
assert component_build.state == koji.BUILD_STATES["BUILDING"]
assert component_build.package == "tangerine"
assert component_build.reused_component_id is not None
mock_sbc.assert_not_called()
@patch("module_build_service.utils.batches.start_build_component")
def test_start_next_batch_build_smart_scheduling(
self, mock_sbc, default_buildroot_groups
):
"""
Tests that components with the longest build time will be scheduled first
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.batch = 1
pt_component = models.ComponentBuild.from_component_name(
db_session, "perl-Tangerine", 3)
pt_component.ref = "6ceea46add2366d8b8c5a623b2fb563b625bfabe"
plc_component = models.ComponentBuild.from_component_name(
db_session, "perl-List-Compare", 3)
plc_component.ref = "5ceea46add2366d8b8c5a623a2fb563b625b9abd"
# Components are by default built by component id. To find out that weight is respected,
# we have to set bigger weight to component with lower id.
pt_component.weight = 3 if pt_component.id < plc_component.id else 4
plc_component.weight = 4 if pt_component.id < plc_component.id else 3
builder = mock.MagicMock()
builder.recover_orphaned_artifact.return_value = []
module_build_service.utils.start_next_batch_build(
conf, module_build, builder)
# Batch number should increase.
assert module_build.batch == 2
# Make sure we don't have any messages returned since no components should be reused
assert len(events.scheduler.queue) == 0
# Make sure both components are set to the build state but not reused
assert pt_component.state == koji.BUILD_STATES["BUILDING"]
assert pt_component.reused_component_id is None
assert plc_component.state == koji.BUILD_STATES["BUILDING"]
assert plc_component.reused_component_id is None
# Test the order of the scheduling
expected_calls = [
mock.call(db_session, builder, plc_component),
mock.call(db_session, builder, pt_component)
]
assert mock_sbc.mock_calls == expected_calls
@patch("module_build_service.utils.batches.start_build_component")
def test_start_next_batch_continue(self, mock_sbc, default_buildroot_groups):
"""
Tests that start_next_batch_build does not start new batch when
there are unbuilt components in the current one.
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.batch = 2
# The component was reused when the batch first started
building_component = module_build.current_batch()[0]
building_component.state = koji.BUILD_STATES["BUILDING"]
db_session.commit()
builder = mock.MagicMock()
module_build_service.utils.start_next_batch_build(
conf, module_build, builder)
# Batch number should not increase.
assert module_build.batch == 2
# Make sure start build was called for the second component which wasn't reused
mock_sbc.assert_called_once()
# No further work should be returned
assert len(events.scheduler.queue) == 0
def test_start_next_batch_build_repo_building(self, default_buildroot_groups):
"""
Test that start_next_batch_build does not start new batch when
builder.buildroot_ready() returns False.
"""
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.batch = 1
db_session.commit()
builder = mock.MagicMock()
builder.buildroot_ready.return_value = False
# Batch number should not increase.
assert module_build.batch == 1
@patch(
"module_build_service.config.Config.mock_resultsdir",
new_callable=mock.PropertyMock,