diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index 1f914d12..e4e2f220 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -337,7 +337,14 @@ def wait(config, session, msg): component_build.reason = reason component_build.nvr = nvr elif component_build.state != koji.BUILD_STATES['COMPLETE']: - task_id, state, reason, nvr = builder.build(artifact_name=artifact_name, source=srpm) + # It's possible that the build succeeded in the builder but some other step failed which + # caused module-build-macros to be marked as failed in MBS, so check to see if it exists + # first + msgs = builder.recover_orphaned_artifact(component_build) + if msgs: + further_work += msgs + else: + task_id, state, reason, nvr = builder.build(artifact_name=artifact_name, source=srpm) session.add(component_build) build.transition(config, state="build") diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 0cf52dc5..edbed65a 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -991,6 +991,116 @@ class TestBuild(unittest.TestCase): self.assertTrue(build.module_build.state in [models.BUILD_STATES['done'], models.BUILD_STATES['ready']]) + @timed(60) + @patch('module_build_service.auth.get_user', return_value=user) + @patch('module_build_service.scm.SCM') + def test_submit_build_resume_recover_orphaned_macros( + self, mocked_scm, mocked_get_user, conf_system, dbg): + """ + Tests that resuming the build works when module-build-macros is orphaned but marked as + failed in the database + """ + FakeModuleBuilder.INSTANT_COMPLETE = True + now = datetime.utcnow() + submitted_time = now - timedelta(minutes=3) + # Create a module in the failed state + build_one = models.ModuleBuild() + build_one.name = 'testmodule' + build_one.stream = 'master' + build_one.version = 1 + build_one.state = models.BUILD_STATES['failed'] + current_dir = os.path.dirname(__file__) + formatted_testmodule_yml_path = os.path.join( + current_dir, '..', 'staged_data', 'formatted_testmodule.yaml') + with open(formatted_testmodule_yml_path, 'r') as f: + build_one.modulemd = f.read() + build_one.koji_tag = 'module-testmodule-master-1' + build_one.scmurl = 'git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#7fea453' + build_one.batch = 2 + build_one.owner = 'Homer J. Simpson' + build_one.time_submitted = submitted_time + build_one.time_modified = now + build_one.rebuild_strategy = 'changed-and-after' + # It went from init, to wait, to build, and then failed + mbt_one = models.ModuleBuildTrace( + state_time=submitted_time, state=models.BUILD_STATES['init']) + mbt_two = models.ModuleBuildTrace( + state_time=now - timedelta(minutes=2), state=models.BUILD_STATES['wait']) + mbt_three = models.ModuleBuildTrace( + state_time=now - timedelta(minutes=1), state=models.BUILD_STATES['build']) + mbt_four = models.ModuleBuildTrace(state_time=now, state=build_one.state) + build_one.module_builds_trace.append(mbt_one) + build_one.module_builds_trace.append(mbt_two) + build_one.module_builds_trace.append(mbt_three) + build_one.module_builds_trace.append(mbt_four) + # Components that haven't started yet + component_one = models.ComponentBuild() + component_one.package = 'perl-Tangerine' + component_one.format = 'rpms' + component_one.scmurl = 'git://pkgs.stg.fedoraproject.org/rpms/perl-Tangerine.git?#f24' + component_one.batch = 2 + component_one.module_id = 1 + component_two = models.ComponentBuild() + component_two.package = 'perl-List-Compare' + component_two.format = 'rpms' + component_two.scmurl = 'git://pkgs.stg.fedoraproject.org/rpms/perl-List-Compare.git?#f24' + component_two.batch = 2 + component_two.module_id = 1 + component_three = models.ComponentBuild() + component_three.package = 'tangerine' + component_three.format = 'rpms' + component_three.scmurl = 'git://pkgs.stg.fedoraproject.org/rpms/tangerine.git?#f24' + component_three.batch = 3 + component_three.module_id = 1 + # Failed module-build-macros + component_four = models.ComponentBuild() + component_four.package = 'module-build-macros' + component_four.format = 'rpms' + component_four.state = koji.BUILD_STATES['FAILED'] + component_four.scmurl = ( + '/tmp/module_build_service-build-macrosqr4AWH/SRPMS/module-build-macros-0.1-1.' + 'module_testmodule_master_20170109091357.src.rpm') + component_four.batch = 1 + component_four.module_id = 1 + component_four.build_time_only = True + + db.session.add(build_one) + db.session.add(component_one) + db.session.add(component_two) + db.session.add(component_three) + db.session.add(component_four) + db.session.commit() + db.session.expire_all() + + FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml', '7fea453') + # Resubmit the failed module + rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( + {'branch': 'master', 'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' + 'testmodule.git?#7fea453'})) + + data = json.loads(rv.data) + module_build_id = data['id'] + module_build = models.ModuleBuild.query.filter_by(id=module_build_id).one() + # Make sure the build went from failed to wait + self.assertEqual(module_build.state, models.BUILD_STATES['wait']) + self.assertEqual(module_build.state_reason, 'Resubmitted by Homer J. Simpson') + # Make sure the state was reset on the failed component + for c in module_build.component_builds: + self.assertIsNone(c.state) + db.session.expire_all() + + # Run the backend + msgs = [] + stop = module_build_service.scheduler.make_simple_stop_condition(db.session) + module_build_service.scheduler.main(msgs, stop) + + # All components should be built and module itself should be in "done" + # or "ready" state. + for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all(): + self.assertEqual(build.state, koji.BUILD_STATES['COMPLETE']) + self.assertTrue(build.module_build.state in [models.BUILD_STATES['done'], + models.BUILD_STATES['ready']]) + @timed(60) @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM')