diff --git a/conf/config.py b/conf/config.py index 09f7739f..6ee310c2 100644 --- a/conf/config.py +++ b/conf/config.py @@ -46,7 +46,7 @@ class BaseConfiguration(object): # Determines how many builds that can be submitted to the builder # and be in the build state at a time. Set this to 0 for no restrictions - NUM_CONSECUTIVE_BUILDS = 5 + NUM_CONCURRENT_BUILDS = 5 ALLOW_CUSTOM_SCMURLS = False diff --git a/module_build_service/config.py b/module_build_service/config.py index 345e43b6..f0b0eeee 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -317,10 +317,10 @@ class Config(object): 'type': bool, 'default': False, 'desc': 'Is it allowed to directly submit modulemd yaml file?'}, - 'num_consecutive_builds': { + 'num_concurrent_builds': { 'type': int, 'default': 0, - 'desc': 'Number of consecutive component builds.'}, + 'desc': 'Number of concurrent component builds.'}, 'net_timeout': { 'type': int, 'default': 120, @@ -463,9 +463,9 @@ class Config(object): raise TypeError("scmurls needs to be a list.") self._scmurls = [str(x) for x in l] - def _setifok_num_consecutive_builds(self, i): + def _setifok_num_concurrent_builds(self, i): if not isinstance(i, int): - raise TypeError('NUM_CONSECUTIVE_BUILDS needs to be an int') + raise TypeError('NUM_CONCURRENT_BUILDS needs to be an int') if i < 0: - raise ValueError('NUM_CONSECUTIVE_BUILDS must be >= 0') - self._num_consecutive_builds = i + raise ValueError('NUM_CONCURRENT_BUILDS must be >= 0') + self._num_concurrent_builds = i diff --git a/module_build_service/utils.py b/module_build_service/utils.py index 99ca4cca..a47b080a 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -89,13 +89,13 @@ def at_concurrent_component_threshold(config, session): # build whole module in this single continue_batch_build call to keep # the number of created buildroots low. The concurrent build limit # for mock backend is secured by setting max_workers in - # ThreadPoolExecutor to num_consecutive_builds. + # ThreadPoolExecutor to num_concurrent_builds. if conf.system == "mock": return False import koji # Placed here to avoid py2/py3 conflicts... - if config.num_consecutive_builds and config.num_consecutive_builds <= \ + if config.num_concurrent_builds and config.num_concurrent_builds <= \ session.query(models.ComponentBuild).filter_by( state=koji.BUILD_STATES['BUILDING']).count(): return True @@ -148,7 +148,7 @@ def continue_batch_build(config, module, session, builder, components=None): # Get the list of components to be build in this batch. We are not # building all `unbuilt_components`, because we can a) meet - # the num_consecutive_builds threshold or b) reuse previous build. + # the num_concurrent_builds threshold or b) reuse previous build. further_work = [] components_to_build = [] for c in unbuilt_components: @@ -175,8 +175,8 @@ def continue_batch_build(config, module, session, builder, components=None): # Start build of components in this batch. max_workers = 1 - if config.num_consecutive_builds > 0: - max_workers = config.num_consecutive_builds + if config.num_concurrent_builds > 0: + max_workers = config.num_concurrent_builds with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: futures = {executor.submit(start_build_component, builder, c): c for c in components_to_build} concurrent.futures.wait(futures) diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 4e9b47f6..321bf91e 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -432,14 +432,14 @@ class TestBuild(unittest.TestCase): @timed(30) @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') - @patch("module_build_service.config.Config.num_consecutive_builds", + @patch("module_build_service.config.Config.num_concurrent_builds", new_callable=PropertyMock, return_value=1) - def test_submit_build_concurrent_threshold(self, conf_num_consecutive_builds, + def test_submit_build_concurrent_threshold(self, conf_num_concurrent_builds, mocked_scm, mocked_get_user, conf_system, dbg): """ Tests the build of testmodule.yaml using TestModuleBuilder with - num_consecutive_builds set to 1. + num_concurrent_builds set to 1. """ MockedSCM(mocked_scm, 'testmodule', 'testmodule.yaml', '620ec77321b2ea7b0d67d82992dda3e1d67055b4') @@ -454,10 +454,10 @@ class TestBuild(unittest.TestCase): def stop(message): """ Stop the scheduler when the module is built or when we try to build - more components than the num_consecutive_builds. + more components than the num_concurrent_builds. """ main_stop = module_build_service.scheduler.make_simple_stop_condition(db.session) - over_threshold = conf.num_consecutive_builds < \ + over_threshold = conf.num_concurrent_builds < \ db.session.query(models.ComponentBuild).filter_by( state=koji.BUILD_STATES['BUILDING']).count() return main_stop(message) or over_threshold @@ -469,22 +469,22 @@ class TestBuild(unittest.TestCase): # or "ready" state. for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all(): self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE']) - # When this fails, it can mean that num_consecutive_builds + # When this fails, it can mean that num_concurrent_builds # threshold has been met. self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] ) @timed(30) @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') - @patch("module_build_service.config.Config.num_consecutive_builds", + @patch("module_build_service.config.Config.num_concurrent_builds", new_callable=PropertyMock, return_value=2) - def test_try_to_reach_concurrent_threshold(self, conf_num_consecutive_builds, + def test_try_to_reach_concurrent_threshold(self, conf_num_concurrent_builds, mocked_scm, mocked_get_user, conf_system, dbg): """ Tests that we try to submit new component build right after the previous one finished without waiting for all - the num_consecutive_builds to finish. + the num_concurrent_builds to finish. """ MockedSCM(mocked_scm, 'testmodule-more-components', 'testmodule-more-components.yaml', '620ec77321b2ea7b0d67d82992dda3e1d67055b4') @@ -503,12 +503,12 @@ class TestBuild(unittest.TestCase): def stop(message): """ Stop the scheduler when the module is built or when we try to build - more components than the num_consecutive_builds. + more components than the num_concurrent_builds. """ main_stop = module_build_service.scheduler.make_simple_stop_condition(db.session) num_building = db.session.query(models.ComponentBuild).filter_by( state=koji.BUILD_STATES['BUILDING']).count() - over_threshold = conf.num_consecutive_builds < num_building + over_threshold = conf.num_concurrent_builds < num_building TestBuild._global_var.append(num_building) return main_stop(message) or over_threshold @@ -523,7 +523,7 @@ class TestBuild(unittest.TestCase): # when we should be building just single component: # 1) module-base-macros in first batch. # 2) The last component of second batch. - # If we are building single component more often, num_consecutive_builds + # If we are building single component more often, num_concurrent_builds # does not work correctly. num_builds = [k for k, g in itertools.groupby(TestBuild._global_var)] self.assertEqual(num_builds.count(1), 2) @@ -531,9 +531,9 @@ class TestBuild(unittest.TestCase): @timed(30) @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') - @patch("module_build_service.config.Config.num_consecutive_builds", + @patch("module_build_service.config.Config.num_concurrent_builds", new_callable=PropertyMock, return_value=1) - def test_build_in_batch_fails(self, conf_num_consecutive_builds, mocked_scm, + def test_build_in_batch_fails(self, conf_num_concurrent_builds, mocked_scm, mocked_get_user, conf_system, dbg): """ Tests that if the build in batch fails, other components in a batch @@ -591,9 +591,9 @@ class TestBuild(unittest.TestCase): @timed(30) @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') - @patch("module_build_service.config.Config.num_consecutive_builds", + @patch("module_build_service.config.Config.num_concurrent_builds", new_callable=PropertyMock, return_value=1) - def test_all_builds_in_batch_fail(self, conf_num_consecutive_builds, mocked_scm, + def test_all_builds_in_batch_fail(self, conf_num_concurrent_builds, mocked_scm, mocked_get_user, conf_system, dbg): """ Tests that if the build in batch fails, other components in a batch