From 3318f48fc31209608c3ac48e117ae6953256e62b Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Fri, 17 Mar 2017 08:42:37 +0100 Subject: [PATCH] Fix the process_paused_module_builds in poller --- module_build_service/builder.py | 18 +++++ module_build_service/models.py | 2 +- module_build_service/scheduler/producer.py | 18 ++++- tests/test_scheduler/test_poller.py | 90 ++++++++++++++++++++++ 4 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 tests/test_scheduler/test_poller.py diff --git a/module_build_service/builder.py b/module_build_service/builder.py index e2b29148..80d99b45 100644 --- a/module_build_service/builder.py +++ b/module_build_service/builder.py @@ -138,6 +138,24 @@ class GenericBuilder(six.with_metaclass(ABCMeta)): else: raise ValueError("Builder backend='%s' not recognized" % backend) + @classmethod + def create_from_module(cls, session, module, config): + """ + Creates new GenericBuilder instance based on the data from module + and config and connects it to buildroot. + + :param session: SQLAlchemy databa session. + :param module: module_build_service.models.ModuleBuild instance. + :param config: module_build_service.config.Config instance. + """ + components = [c.package for c in module.component_builds] + builder = GenericBuilder.create( + module.owner, module.name, config.system, config, + tag_name=module.koji_tag, components=components) + groups = GenericBuilder.default_buildroot_groups(session, module) + builder.buildroot_connect(groups) + return builder + @classmethod def tag_to_repo(cls, backend, config, tag_name, arch): """ diff --git a/module_build_service/models.py b/module_build_service/models.py index 962a09e6..b7542560 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -121,7 +121,7 @@ class ModuleBuild(MBSBase): if not self.batch: raise ValueError("No batch is in progress: %r" % self.batch) - if state: + if state != None: return [ component for component in self.component_builds if component.batch == self.batch and component.state == state diff --git a/module_build_service/scheduler/producer.py b/module_build_service/scheduler/producer.py index b269ed89..116d035d 100644 --- a/module_build_service/scheduler/producer.py +++ b/module_build_service/scheduler/producer.py @@ -34,6 +34,7 @@ import module_build_service.messaging import module_build_service.scheduler import module_build_service.scheduler.consumer from module_build_service import conf, models, log +from module_build_service.builder import GenericBuilder class MBSProducer(PollingProducer): @@ -137,7 +138,8 @@ class MBSProducer(PollingProducer): if name == 'build': for module_build in query.all(): log.info(' * {0!r}'.format(module_build)) - for i in range(module_build.batch): + # First batch is number '1'. + for i in range(1, module_build.batch + 1): n = len([c for c in module_build.component_builds if c.batch == i ]) log.info(' * {0} components in batch {1}' @@ -170,6 +172,7 @@ class MBSProducer(PollingProducer): log.debug('Will not attempt to start paused module builds due to ' 'the concurrent build threshold being met') return + # Check to see if module builds that are in build state but don't have # any component builds being built can be worked on for module_build in session.query(models.ModuleBuild).filter_by( @@ -177,8 +180,17 @@ class MBSProducer(PollingProducer): # If there are no components in the build state on the module build, # then no possible event will start off new component builds if not module_build.current_batch(koji.BUILD_STATES['BUILDING']): - further_work = module_build_service.utils.start_build_batch( - config, module_build, session, config.system) + # Initialize the builder... + builder = GenericBuilder.create_from_module( + session, module_build, config) + + further_work = module_build_service.utils.start_next_batch_build( + config, module_build, session, builder) for event in further_work: log.info(" Scheduling faked event %r" % event) module_build_service.scheduler.consumer.work_queue_put(event) + + # Check if we have met the threshold. + if module_build_service.utils.at_concurrent_component_threshold( + config, session): + break diff --git a/tests/test_scheduler/test_poller.py b/tests/test_scheduler/test_poller.py new file mode 100644 index 00000000..71bf15fd --- /dev/null +++ b/tests/test_scheduler/test_poller.py @@ -0,0 +1,90 @@ +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import unittest +from os import path +import vcr +import modulemd +from mock import patch +import module_build_service.utils +import module_build_service.scm +from module_build_service import models, conf +from module_build_service.errors import ProgrammingError, ValidationError +from tests import test_reuse_component_init_data, init_data, db +import mock +from mock import PropertyMock +import koji +import module_build_service.scheduler.handlers.components +from module_build_service.builder import GenericBuilder, KojiModuleBuilder +from module_build_service.scheduler.producer import MBSProducer +import six.moves.queue as queue + +BASE_DIR = path.abspath(path.dirname(__file__)) +CASSETTES_DIR = path.join( + path.abspath(path.dirname(__file__)), '..', 'vcr-request-data') + +@patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", + return_value = {'build': [], 'srpm-build': []}) +@patch("module_build_service.scheduler.consumer.get_global_consumer") +@patch("module_build_service.builder.KojiModuleBuilder.get_session") +@patch("module_build_service.builder.GenericBuilder.create_from_module") +class TestPoller(unittest.TestCase): + + def setUp(self): + test_reuse_component_init_data() + + def tearDown(self): + init_data() + + def test_process_paused_module_builds(self, crete_builder, + koji_get_session, global_consumer, + dbg): + """ + Tests general use-case of process_paused_module_builds. + """ + consumer = mock.MagicMock() + consumer.incoming = queue.Queue() + global_consumer.return_value = consumer + + koji_session = mock.MagicMock() + koji_get_session.return_value = koji_session + + builder = mock.MagicMock() + crete_builder.return_value = builder + + # Change the batch to 2, so the module build is in state where + # it is not building anything, but the state is "build". + module_build = models.ModuleBuild.query.filter_by(id=2).one() + module_build.batch = 2 + db.session.commit() + + # Poll :) + hub = mock.MagicMock() + poller = MBSProducer(hub) + poller.poll() + + # Refresh our module_build object. + db.session.expunge(module_build) + module_build = models.ModuleBuild.query.filter_by(id=2).one() + + # Components should be in BUILDING state now. + components = module_build.current_batch() + for component in components: + self.assertEqual(component.state, koji.BUILD_STATES["BUILDING"])