From 999baa2992cd59b022acb8ef3bf8f64a1eb9b3b8 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Fri, 15 Sep 2017 11:46:37 +0200 Subject: [PATCH] Wait for components to be tagged also in final tag before marking module as done. --- .../migrations/versions/c11a3cfec2a9_.py | 24 ++++ module_build_service/models.py | 7 +- .../scheduler/handlers/components.py | 5 + .../scheduler/handlers/tags.py | 16 +-- tests/test_scheduler/test_tag_tagged.py | 118 ++++++++++++++++++ 5 files changed, 162 insertions(+), 8 deletions(-) create mode 100644 module_build_service/migrations/versions/c11a3cfec2a9_.py diff --git a/module_build_service/migrations/versions/c11a3cfec2a9_.py b/module_build_service/migrations/versions/c11a3cfec2a9_.py new file mode 100644 index 00000000..53a1e913 --- /dev/null +++ b/module_build_service/migrations/versions/c11a3cfec2a9_.py @@ -0,0 +1,24 @@ +"""Add component_builds.build_time_only and component_builds.tagged_in_final. + +Revision ID: c11a3cfec2a9 +Revises: 3b17cd6dc583 +Create Date: 2017-09-15 15:23:55.357689 + +""" + +# revision identifiers, used by Alembic. +revision = 'c11a3cfec2a9' +down_revision = '3b17cd6dc583' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('component_builds', sa.Column('build_time_only', sa.Boolean(), nullable=True)) + op.add_column('component_builds', sa.Column('tagged_in_final', sa.Boolean(), nullable=True)) + + +def downgrade(): + op.drop_column('component_builds', 'tagged_in_final') + op.drop_column('component_builds', 'build_time_only') diff --git a/module_build_service/models.py b/module_build_service/models.py index 4ed75cf1..36d6fe54 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -454,8 +454,13 @@ class ComponentBuild(MBSBase): state_reason = db.Column(db.String) # This stays as None until the build completes. nvr = db.Column(db.String) - # True when this component build is tagged into buildroot. + # True when this component build is tagged into buildroot (-build tag). tagged = db.Column(db.Boolean, default=False) + # True when this component build is tagged into final tag. + tagged_in_final = db.Column(db.Boolean, default=False) + # True when this component build is build-time only (should be tagged only + # to -build tag) + build_time_only = db.Column(db.Boolean, default=False) # A monotonically increasing integer that represents which batch or # iteration this *component* is currently in. This relates to the owning diff --git a/module_build_service/scheduler/handlers/components.py b/module_build_service/scheduler/handlers/components.py index 3a6788a7..5fb49087 100644 --- a/module_build_service/scheduler/handlers/components.py +++ b/module_build_service/scheduler/handlers/components.py @@ -124,6 +124,11 @@ def _finalize(config, session, msg, state): # Do not tag packages which belong to -build tag to final tag. if not install: builder.tag_artifacts(built_components_in_batch) + else: + # For components which should not be tagged in final Koji tag, + # set the build_time_only to True so we do not wait for the + # non-existing tagging task to finish. + component_build.build_time_only = True session.commit() elif (any([c.state != koji.BUILD_STATES['BUILDING'] diff --git a/module_build_service/scheduler/handlers/tags.py b/module_build_service/scheduler/handlers/tags.py index bdf7d78f..18ba8954 100644 --- a/module_build_service/scheduler/handlers/tags.py +++ b/module_build_service/scheduler/handlers/tags.py @@ -40,9 +40,6 @@ def tagged(config, session, msg): # Find our ModuleBuild associated with this tagged artifact. tag = msg.tag - if not tag.endswith('-build'): - log.debug("Tag %r does not end with '-build' suffix, ignoring" % tag) - return module_build = models.ModuleBuild.from_tag_change_event(session, msg) if not module_build: log.debug("No module build found associated with koji tag %r" % tag) @@ -59,7 +56,10 @@ def tagged(config, session, msg): msg.msg_id)) # Mark the component as tagged - component.tagged = True + if tag.endswith('-build'): + component.tagged = True + else: + component.tagged_in_final = True session.commit() unbuilt_components_in_batch = [ @@ -75,7 +75,8 @@ def tagged(config, session, msg): # have been built successfully. untagged_components = [ c for c in module_build.up_to_current_batch() - if not c.tagged and c.state == koji.BUILD_STATES['COMPLETE'] + if (not c.tagged or (not c.tagged_in_final and not c.build_time_only)) and + c.state == koji.BUILD_STATES['COMPLETE'] ] further_work = [] @@ -90,8 +91,9 @@ def tagged(config, session, msg): if c.state == koji.BUILD_STATES['BUILDING'] or not c.state ] if unbuilt_components: - log.info("All components in batch tagged, regenerating repo for tag %s", tag) - task_id = builder.koji_session.newRepo(tag) + repo_tag = builder.module_build_tag['name'] + log.info("All components in batch tagged, regenerating repo for tag %s", repo_tag) + task_id = builder.koji_session.newRepo(repo_tag) module_build.new_repo_task_id = task_id else: # In case this is the last batch, we do not need to regenerate the diff --git a/tests/test_scheduler/test_tag_tagged.py b/tests/test_scheduler/test_tag_tagged.py index f5abe7ba..0e7d12dd 100644 --- a/tests/test_scheduler/test_tag_tagged.py +++ b/tests/test_scheduler/test_tag_tagged.py @@ -79,6 +79,7 @@ class TestTagTagged(unittest.TestCase): builder = mock.MagicMock() builder.koji_session = koji_session builder.buildroot_ready.return_value = False + builder.module_build_tag = {"name": "module-testmodule-build"} create_builder.return_value = builder module_build = module_build_service.models.ModuleBuild.query.filter_by(id=2).one() @@ -88,6 +89,7 @@ class TestTagTagged(unittest.TestCase): for c in module_build.up_to_current_batch(): c.state = koji.BUILD_STATES["COMPLETE"] c.tagged = True + c.tagged_in_final = True module_build.batch = 2 for c in module_build.current_batch(): @@ -99,6 +101,11 @@ class TestTagTagged(unittest.TestCase): 'id', 'module-testmodule-build', "perl-Tangerine") module_build_service.scheduler.handlers.tags.tagged( config=conf, session=db.session, msg=msg) + # Tag the first component to the final tag. + msg = module_build_service.messaging.KojiTagChange( + 'id', 'module-testmodule', "perl-Tangerine") + module_build_service.scheduler.handlers.tags.tagged( + config=conf, session=db.session, msg=msg) # newRepo should not be called, because there are still components # to tag. @@ -110,6 +117,16 @@ class TestTagTagged(unittest.TestCase): module_build_service.scheduler.handlers.tags.tagged( config=conf, session=db.session, msg=msg) + # newRepo should not be called, because the component has not been + # tagged to final tag so far. + self.assertTrue(not koji_session.newRepo.called) + + # Tag the first component to the final tag. + msg = module_build_service.messaging.KojiTagChange( + 'id', 'module-testmodule', "perl-List-Compare") + module_build_service.scheduler.handlers.tags.tagged( + config=conf, session=db.session, msg=msg) + # newRepo should be called now - all components have been tagged. koji_session.newRepo.assert_called_once_with("module-testmodule-build") @@ -138,6 +155,7 @@ class TestTagTagged(unittest.TestCase): builder = mock.MagicMock() builder.koji_session = koji_session builder.buildroot_ready.return_value = False + builder.module_build_tag = {"name": "module-testmodule-build"} create_builder.return_value = builder module_build = module_build_service.models.ModuleBuild.query.filter_by(id=2).one() @@ -152,6 +170,11 @@ class TestTagTagged(unittest.TestCase): 'id', 'module-testmodule-build', "perl-Tangerine") module_build_service.scheduler.handlers.tags.tagged( config=conf, session=db.session, msg=msg) + # Tag the perl-List-Compare component to final tag. + msg = module_build_service.messaging.KojiTagChange( + 'id', 'module-testmodule', "perl-Tangerine") + module_build_service.scheduler.handlers.tags.tagged( + config=conf, session=db.session, msg=msg) # newRepo should not be called, because perl-List-Compare has not been # built yet. @@ -174,6 +197,7 @@ class TestTagTagged(unittest.TestCase): builder = mock.MagicMock() builder.koji_session = koji_session builder.buildroot_ready.return_value = False + builder.module_build_tag = {"name": "module-testmodule-build"} create_builder.return_value = builder module_build = module_build_service.models.ModuleBuild.query.filter_by(id=2).one() @@ -183,6 +207,7 @@ class TestTagTagged(unittest.TestCase): for c in module_build.up_to_current_batch(): c.state = koji.BUILD_STATES["COMPLETE"] c.tagged = True + c.tagged_in_final = True module_build.batch = 2 component = module_build_service.models.ComponentBuild.query\ @@ -198,6 +223,11 @@ class TestTagTagged(unittest.TestCase): 'id', 'module-testmodule-build', "perl-List-Compare") module_build_service.scheduler.handlers.tags.tagged( config=conf, session=db.session, msg=msg) + # Tag the perl-List-Compare component to final tag. + msg = module_build_service.messaging.KojiTagChange( + 'id', 'module-testmodule', "perl-List-Compare") + module_build_service.scheduler.handlers.tags.tagged( + config=conf, session=db.session, msg=msg) # newRepo should be called now - all successfully built # components have been tagged. @@ -231,6 +261,7 @@ class TestTagTagged(unittest.TestCase): builder = mock.MagicMock() builder.koji_session = koji_session builder.buildroot_ready.return_value = False + builder.module_build_tag = {"name": "module-testmodule-build"} create_builder.return_value = builder module_build = module_build_service.models.ModuleBuild.query.filter_by(id=2).one() @@ -244,6 +275,11 @@ class TestTagTagged(unittest.TestCase): 'id', 'module-testmodule-build', "perl-Tangerine") module_build_service.scheduler.handlers.tags.tagged( config=conf, session=db.session, msg=msg) + # Tag the first component to the final tag. + msg = module_build_service.messaging.KojiTagChange( + 'id', 'module-testmodule', "perl-Tangerine") + module_build_service.scheduler.handlers.tags.tagged( + config=conf, session=db.session, msg=msg) # newRepo should not be called, because there are still components # to tag. @@ -254,6 +290,11 @@ class TestTagTagged(unittest.TestCase): 'id', 'module-testmodule-build', "perl-List-Compare") module_build_service.scheduler.handlers.tags.tagged( config=conf, session=db.session, msg=msg) + # Tag the second component to final tag. + msg = module_build_service.messaging.KojiTagChange( + 'id', 'module-testmodule', "perl-List-Compare") + module_build_service.scheduler.handlers.tags.tagged( + config=conf, session=db.session, msg=msg) # newRepo should not be called, because there are still components # to tag. @@ -264,6 +305,11 @@ class TestTagTagged(unittest.TestCase): 'id', 'module-testmodule-build', "module-build-macros") module_build_service.scheduler.handlers.tags.tagged( config=conf, session=db.session, msg=msg) + # Tag the component from first batch to final tag. + msg = module_build_service.messaging.KojiTagChange( + 'id', 'module-testmodule', "module-build-macros") + module_build_service.scheduler.handlers.tags.tagged( + config=conf, session=db.session, msg=msg) # newRepo should be called now - all components have been tagged. koji_session.newRepo.assert_called_once_with("module-testmodule-build") @@ -275,3 +321,75 @@ class TestTagTagged(unittest.TestCase): # newRepo task_id should be stored in database, so we can check its # status later in poller. self.assertEqual(module_build.new_repo_task_id, 123456) + + + @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", + return_value={'build': [], 'srpm-build': []}) + @patch("module_build_service.builder.KojiModuleBuilder.get_session") + @patch("module_build_service.builder.GenericBuilder.create_from_module") + def test_newrepo_build_time_only( + self, create_builder, koji_get_session, dbg): + """ + Test the component.build_time_only is respected in tag handler. + """ + koji_session = mock.MagicMock() + koji_session.getTag = lambda tag_name: {'name': tag_name} + koji_session.getTaskInfo.return_value = {'state': koji.TASK_STATES['CLOSED']} + koji_session.newRepo.return_value = 123456 + koji_get_session.return_value = koji_session + + builder = mock.MagicMock() + builder.koji_session = koji_session + builder.buildroot_ready.return_value = False + builder.module_build_tag = {"name": "module-testmodule-build"} + create_builder.return_value = builder + + module_build = module_build_service.models.ModuleBuild.query.filter_by(id=2).one() + + # Set previous components as COMPLETE and tagged. + module_build.batch = 1 + for c in module_build.up_to_current_batch(): + c.state = koji.BUILD_STATES["COMPLETE"] + c.tagged = True + c.tagged_in_final = True + + module_build.batch = 2 + component = module_build_service.models.ComponentBuild.query\ + .filter_by(package='perl-Tangerine', module_id=module_build.id).one() + component.state = koji.BUILD_STATES["COMPLETE"] + component.build_time_only = True + component.tagged = False + component.tagged_in_final = False + component = module_build_service.models.ComponentBuild.query\ + .filter_by(package='perl-List-Compare', module_id=module_build.id).one() + component.state = koji.BUILD_STATES["COMPLETE"] + db.session.commit() + + # Tag the perl-Tangerine component to the buildroot. + msg = module_build_service.messaging.KojiTagChange( + 'id', 'module-testmodule-build', "perl-Tangerine") + module_build_service.scheduler.handlers.tags.tagged( + config=conf, session=db.session, msg=msg) + self.assertTrue(not koji_session.newRepo.called) + # Tag the perl-List-Compare component to the buildroot. + msg = module_build_service.messaging.KojiTagChange( + 'id', 'module-testmodule-build', "perl-List-Compare") + module_build_service.scheduler.handlers.tags.tagged( + config=conf, session=db.session, msg=msg) + # Tag the perl-List-Compare component to final tag. + msg = module_build_service.messaging.KojiTagChange( + 'id', 'module-testmodule', "perl-List-Compare") + module_build_service.scheduler.handlers.tags.tagged( + config=conf, session=db.session, msg=msg) + + # newRepo should be called now - all successfully built + # components have been tagged. + koji_session.newRepo.assert_called_once_with("module-testmodule-build") + + # Refresh our module_build object. + db.session.expunge(module_build) + module_build = module_build_service.models.ModuleBuild.query.filter_by(id=2).one() + + # newRepo task_id should be stored in database, so we can check its + # status later in poller. + self.assertEqual(module_build.new_repo_task_id, 123456)