From 23802418afda9a602fe30990f33d92814b28414b Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Fri, 6 Jan 2017 11:04:54 +0100 Subject: [PATCH] Fix tag_artifacts with concurrent build threshold, do not send repo-done in Mock backend after each build, but just after the tag_artifacts call. --- module_build_service/builder.py | 20 +++----- .../scheduler/handlers/components.py | 49 ++++++++++++------- tests/test_build/test_build.py | 40 +++++++++++++-- 3 files changed, 75 insertions(+), 34 deletions(-) diff --git a/module_build_service/builder.py b/module_build_service/builder.py index 7d9a6a2b..ac484318 100644 --- a/module_build_service/builder.py +++ b/module_build_service/builder.py @@ -1185,8 +1185,15 @@ $repos _execute_cmd(["mock", "-r", self.mock_config, "-i", "module-build-macros"]) + def _send_repo_done(self): + msg = module_build_service.messaging.KojiRepoChange( + msg_id='a faked internal message', + repo_tag=self.tag_name + "-build", + ) + module_build_service.scheduler.consumer.work_queue_put(msg) + def tag_artifacts(self, artifacts): - pass + self._send_repo_done() def buildroot_add_repos(self, dependencies): # TODO: We support only dependencies from Koji here. This should be @@ -1196,13 +1203,6 @@ $repos self._add_repo(tag, baseurl) self._write_mock_config() - def _send_repo_done(self): - msg = module_build_service.messaging.KojiRepoChange( - msg_id='a faked internal message', - repo_tag=self.tag_name + "-build", - ) - module_build_service.scheduler.consumer.work_queue_put(msg) - def _send_build_change(self, state, source, build_id): nvr = kobo.rpmlib.parse_nvr(source) @@ -1244,10 +1244,8 @@ $repos # are put in the scheduler's work queue and are handled # by MBS after the build_srpm() method returns and scope gets # back to scheduler.main.main() method. - self._send_repo_done() self._send_build_change(koji.BUILD_STATES['COMPLETE'], source, MockModuleBuilder._build_id) - self._send_repo_done() with open(os.path.join(self.resultsdir, "status.log"), 'w') as f: f.write("complete\n") @@ -1259,10 +1257,8 @@ $repos # are put in the scheduler's work queue and are handled # by MBS after the build_srpm() method returns and scope gets # back to scheduler.main.main() method. - self._send_repo_done() self._send_build_change(koji.BUILD_STATES['FAILED'], source, MockModuleBuilder._build_id) - self._send_repo_done() with open(os.path.join(self.resultsdir, "status.log"), 'w') as f: f.write("failed\n") diff --git a/module_build_service/scheduler/handlers/components.py b/module_build_service/scheduler/handlers/components.py index 6ed790d1..7ebab34b 100644 --- a/module_build_service/scheduler/handlers/components.py +++ b/module_build_service/scheduler/handlers/components.py @@ -71,6 +71,27 @@ def _finalize(config, session, msg, state): session.commit() return + # Initialize the builder, we will need it later. + module_name = parent.name + tag = parent.koji_tag + builder = module_build_service.builder.GenericBuilder.create( + parent.owner, module_name, config.system, config, tag_name=tag) + + try: + groups = { + 'build': parent.resolve_profiles(session, 'buildroot'), + 'srpm-build': parent.resolve_profiles(session, 'srpm-buildroot'), + } + except ValueError: + reason = "Failed to gather buildroot groups from SCM." + log.exception(reason) + parent.transition(config, state=models.BUILD_STATES["failed"], + state_reason=reason) + session.commit() + raise + + builder.buildroot_connect(groups) + # If there are no other components still building in a batch, # we can tag all successfully built components in the batch. unbuilt_components_in_batch = [ @@ -83,30 +104,20 @@ def _finalize(config, session, msg, state): if c.state == koji.BUILD_STATES['COMPLETE'] ] - module_name = parent.name - tag = parent.koji_tag - builder = module_build_service.builder.GenericBuilder.create( - parent.owner, module_name, config.system, config, tag_name=tag) - - try: - groups = { - 'build': parent.resolve_profiles(session, 'buildroot'), - 'srpm-build': parent.resolve_profiles(session, 'srpm-buildroot'), - } - except ValueError: - reason = "Failed to gather buildroot groups from SCM." - log.exception(reason) - parent.transition(config, state=models.BUILD_STATES["failed"], - state_reason=reason) - session.commit() - raise - - builder.buildroot_connect(groups) # tag && add to srpm-build group if neccessary install = bool(component_build.package == 'module-build-macros') builder.buildroot_add_artifacts(built_components_in_batch, install=install) builder.tag_artifacts(built_components_in_batch) session.commit() + else: + # We have some unbuilt components in this batch. We might hit the + # concurrent builds threshold in previous call of start_build_batch + # done in repos.py:done(...), but because we have just finished one + # build, try to call start_build_batch again so in case we hit the + # threshold previously, we will submit another build from this batch. + further_work = module_build_service.utils.start_build_batch( + config, parent, session, builder) + return further_work def complete(config, session, msg): diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 1e3e35d7..da9ec056 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -120,6 +120,7 @@ class TestModuleBuilder(GenericBuilder): def tag_artifacts(self, artifacts): if TestModuleBuilder.on_tag_artifacts_cb: TestModuleBuilder.on_tag_artifacts_cb(self, artifacts) + self._send_repo_done() @property def module_build_tag(self): @@ -152,11 +153,9 @@ class TestModuleBuilder(GenericBuilder): TestModuleBuilder._build_id += 1 if TestModuleBuilder.BUILD_STATE != "BUILDING": - self._send_repo_done() self._send_build_change( koji.BUILD_STATES[TestModuleBuilder.BUILD_STATE], source, TestModuleBuilder._build_id) - self._send_repo_done() if TestModuleBuilder.on_build_cb: TestModuleBuilder.on_build_cb(self, artifact_name, source) @@ -184,6 +183,9 @@ class TestBuild(unittest.TestCase): def setUp(self): GenericBuilder.register_backend_class(TestModuleBuilder) self.client = app.test_client() + self._prev_system = conf.system + self._prev_num_consencutive_builds = conf.num_consecutive_builds + conf.set_item("system", "mock") init_data() @@ -191,7 +193,8 @@ class TestBuild(unittest.TestCase): models.ComponentBuild.query.delete() def tearDown(self): - conf.set_item("system", "koji") + conf.set_item("system", self._prev_system) + conf.set_item("num_consecutive_builds", self._prev_num_consencutive_builds) TestModuleBuilder.reset() # Necessary to restart the twisted reactor for the next test. @@ -341,3 +344,34 @@ class TestBuild(unittest.TestCase): for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all(): self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE']) self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] ) + + @timed(30) + @patch('module_build_service.auth.get_username', return_value='Homer J. Simpson') + @patch('module_build_service.auth.assert_is_packager') + @patch('module_build_service.scm.SCM') + def test_submit_build_concurrent_threshold( + self, mocked_scm, mocked_assert_is_packager, mocked_get_username): + """ + Tests the build of testmodule.yaml using TestModuleBuilder with + num_consecutive_builds set to 1. + """ + MockedSCM(mocked_scm, "testmodule", "testmodule.yaml") + + conf.set_item("num_consecutive_builds", 1) + + rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( + {'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' + 'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'})) + + data = json.loads(rv.data) + module_build_id = data['id'] + + msgs = [] + stop = module_build_service.scheduler.make_simple_stop_condition(db.session) + module_build_service.scheduler.main(msgs, stop) + + # All components should be built and module itself should be in "done" + # or "ready" state. + for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all(): + self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE']) + self.assertTrue(build.module_build.state in [models.BUILD_STATES["done"], models.BUILD_STATES["ready"]] )