Fix #670 - Tag Content Generator Koji build to special tag based on the base module stream

module_build_service/builder/KojiContentGenerator.py

@@ -359,6 +359,42 @@ class KojiContentGenerator(object):
 
         return serverdir
 
+    def _tag_cg_build(self):
+        """
+        Tags the Content Generator build to module.cg_build_koji_tag.
+        """
+        session = KojiModuleBuilder.get_session(self.config, self.owner)
+
+        tag_name = self.module.cg_build_koji_tag
+        if not tag_name:
+            log.info("%r: Not tagging Content Generator build, no "
+                     "cg_build_koji_tag set", self.module)
+            return
+
+        tag_names_to_try = [tag_name, self.config.koji_cg_default_build_tag]
+        for tag in tag_names_to_try:
+            log.info("Trying %s", tag)
+            tag_info = session.getTag(tag)
+            if tag_info:
+                break
+
+            log.info("%r: Tag %s not found in Koji, trying next one.",
+                     self.module, tag)
+
+        if not tag_info:
+            log.warn("%r:, Not tagging Content Generator build, no "
+                     "available tag found, tried %r", self.module,
+                     tag_names_to_try)
+            return
+
+        build = self._get_build()
+        nvr = "%s-%s-%s" % (build["name"], build["version"], build["release"])
+
+        log.info("Content generator build %s will be tagged as %s in "
+                 "Koji", nvr, tag)
+        session.tagBuild(tag_info["id"], nvr)
+
+
     def koji_import(self):
         """This method imports given module into the configured koji instance as
         a content generator based build
@@ -371,6 +407,7 @@ class KojiContentGenerator(object):
         try:
             serverdir = self._upload_outputs(session, metadata, file_dir)
             build_info = session.CGImport(metadata, serverdir)
+            self._tag_cg_build()
             log.info("Content generator import done.")
             log.debug(json.dumps(build_info, sort_keys=True, indent=4))
 

module_build_service/config.py

@@ -389,7 +389,25 @@ class Config(object):
             'type': str,
             'default': '',
             'desc': ('The distinguished name of the container or organizational unit containing '
-                     'the groups in LDAP')}
+                     'the groups in LDAP')},
+        'base_module_names': {
+            'type': set,
+            'default': set(['platform', 'bootstrap']),
+            'desc': "Set of module names which defines the product version "
+                    "(by their stream) of modules depending on them."},
+        'koji_cg_build_tag_template': {
+            'type': str,
+            'default': "{}-modular-updates-candidate",
+            'desc': "Name of a Koji tag where the top-level Content Generator "
+                    "build is tagged to. The '{}' string is replaced by a "
+                    "stream name of a base module on top of which the "
+                    "module is built."},
+        'koji_cg_default_build_tag': {
+            'type': str,
+            'default': "modular-updates-candidate",
+            'desc': "The name of Koji tag which should be used as fallback "
+                    "when koji_cg_build_tag_template tag is not found in "
+                    "Koji."},
     }
 
     def __init__(self, conf_section_obj):

module_build_service/migrations/versions/edb537dd1e8c_.py

@@ -0,0 +1,22 @@
+"""Add cg_build_koji_tag
+
+Revision ID: edb537dd1e8c
+Revises: c11a3cfec2a9
+Create Date: 2017-09-22 13:50:41.433144
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'edb537dd1e8c'
+down_revision = 'c11a3cfec2a9'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column('module_builds', sa.Column('cg_build_koji_tag', sa.String(), nullable=True))
+
+
+def downgrade():
+    op.drop_column('module_builds', 'cg_build_koji_tag')

module_build_service/models.py

@@ -148,6 +148,8 @@ class ModuleBuild(MBSBase):
     state_reason = db.Column(db.String)
     modulemd = db.Column(db.String, nullable=False)
     koji_tag = db.Column(db.String)  # This gets set after 'wait'
+    # Koji tag to which tag the Content Generator Koji build.
+    cg_build_koji_tag = db.Column(db.String) # This gets set after wait
     copr_owner = db.Column(db.String)
     copr_project = db.Column(db.String)
     scmurl = db.Column(db.String)

module_build_service/pdc.py

@@ -389,7 +389,7 @@ def get_module_build_dependencies(session, module_info, strict=False):
     instance.
     :param strict: Normally this function returns None if no module can be
            found.  If strict=True, then a ValueError is raised.
-    :return final list of koji tags
+    :return dict with koji_tag as a key and ModuleMetadata object as value.
 
     Example minimal module_info:
         {
@@ -402,7 +402,7 @@ def get_module_build_dependencies(session, module_info, strict=False):
     # XXX get definitive list of modules
 
     # This is the set we're going to build up and return.
-    module_tags = set()
+    module_tags = {}
 
     if not isinstance(module_info, modulemd.ModuleMetadata):
         queried_module = get_module(session, module_info, strict=strict)
@@ -430,7 +430,10 @@ def get_module_build_dependencies(session, module_info, strict=False):
         modules = _get_recursively_required_modules(
             session, modified_dep, strict=strict)
         tags = [m["koji_tag"] for m in modules]
-        module_tags = module_tags.union(set(tags))
+        for m in modules:
+            if m["koji_tag"] in module_tags:
+                continue
+            module_tags[m["koji_tag"]] = _extract_modulemd(m["modulemd"])
 
     return module_tags
 

module_build_service/scheduler/handlers/modules.py

@@ -172,13 +172,15 @@ def wait(config, session, msg):
         are going to build. We use private method here to allow "retry"
         on failure.
         """
+        cg_build_koji_tag = conf.koji_cg_default_build_tag
+
         if conf.system != "koji":
             # In case of non-koji backend, we want to get the dependencies
             # of the local module build based on ModuleMetadata, because the
             # local build is not stored in PDC and therefore we cannot query
             # it using the `pdc_query` as for Koji below.
             dependencies = module_build_service.pdc.get_module_build_dependencies(
-                pdc_session, build.mmd(), strict=True)
+                pdc_session, build.mmd(), strict=True).keys()
 
             # We also don't want to get the tag name from the PDC, but just
             # generate it locally instead.
@@ -194,16 +196,28 @@ def wait(config, session, msg):
                 'release': module_info['version'],
             }
             log.info("Getting %s deps from pdc (query %r)" % (module_info['name'], pdc_query))
-            dependencies = module_build_service.pdc.get_module_build_dependencies(
+            deps_dict = module_build_service.pdc.get_module_build_dependencies(
                 pdc_session, pdc_query, strict=True)
+            dependencies = set(deps_dict.keys())
+
+            # Find out the name of Koji tag to which the module's Content
+            # Generator build should be tagged once the build finishes.
+            module_names_streams = {mmd.name:mmd.stream
+                                    for mmd in deps_dict.values()}
+            for base_module_name in conf.base_module_names:
+                if base_module_name in module_names_streams:
+                    cg_build_koji_tag = conf.koji_cg_build_tag_template.format(
+                        module_names_streams[base_module_name])
+                    break
+
             log.info("Getting %s tag from pdc (query %r)" % (module_info['name'], pdc_query))
             tag = module_build_service.pdc.get_module_tag(
                 pdc_session, pdc_query, strict=True)
 
-        return dependencies, tag
+        return dependencies, tag, cg_build_koji_tag
 
     try:
-        dependencies, tag = _get_deps_and_tag()
+        dependencies, tag, cg_build_koji_tag = _get_deps_and_tag()
     except ValueError:
         reason = "Failed to get module info from PDC. Max retries reached."
         log.exception(reason)
@@ -219,6 +233,10 @@ def wait(config, session, msg):
     log.debug("Assigning koji tag=%s to module build" % tag)
     build.koji_tag = tag
 
+    log.debug("Assigning Content Generator build koji tag=%s to module "
+              "build", cg_build_koji_tag)
+    build.cg_build_koji_tag = cg_build_koji_tag
+
     builder = module_build_service.builder.GenericBuilder.create_from_module(
         session, build, config)
 

tests/test_content_generator.py

@@ -32,7 +32,7 @@ import module_build_service.messaging
 import module_build_service.scheduler.handlers.repos
 from module_build_service import models, conf, build_logs
 
-from mock import patch, Mock
+from mock import patch, Mock, MagicMock, call
 
 from tests import init_data
 
@@ -52,6 +52,7 @@ class TestBuild(unittest.TestCase):
     def setUp(self):
         init_data()
         module = models.ModuleBuild.query.filter_by(id=1).one()
+        module.cg_build_koji_tag = "f27-module-candidate"
         self.cg = KojiContentGenerator(module, conf)
 
         filename = cassette_dir + self.id()
@@ -158,3 +159,52 @@ class TestBuild(unittest.TestCase):
         dir_path = self.cg._prepare_file_directory()
         with open(path.join(dir_path, "modulemd.txt")) as mmd:
             self.assertEqual(len(mmd.read()), 1134)
+
+    @patch("module_build_service.builder.KojiModuleBuilder.get_session")
+    def test_tag_cg_build(self, get_session):
+        """ Test that the CG build is tagged. """
+        koji_session = MagicMock()
+        koji_session.getTag.return_value = {'id': 123}
+        get_session.return_value = koji_session
+
+        self.cg._tag_cg_build()
+
+        koji_session.getTag.assert_called_once_with(self.cg.module.cg_build_koji_tag)
+        koji_session.tagBuild.assert_called_once_with(123, "nginx-1-2")
+
+    @patch("module_build_service.builder.KojiModuleBuilder.get_session")
+    def test_tag_cg_build_fallback_to_default_tag(self, get_session):
+        """ Test that the CG build is tagged to default tag. """
+        koji_session = MagicMock()
+        koji_session.getTag.side_effect = [{}, {'id': 123}]
+        get_session.return_value = koji_session
+
+        self.cg._tag_cg_build()
+
+        self.assertEqual(koji_session.getTag.mock_calls,
+                         [call(self.cg.module.cg_build_koji_tag),
+                          call(conf.koji_cg_default_build_tag)])
+        koji_session.tagBuild.assert_called_once_with(123, "nginx-1-2")
+
+    @patch("module_build_service.builder.KojiModuleBuilder.get_session")
+    def test_tag_cg_build_no_tag_set(self, get_session):
+        """ Test that the CG build is not tagged when no tag set. """
+        koji_session = MagicMock()
+        koji_session.getTag.side_effect = [{}, {'id': 123}]
+        get_session.return_value = koji_session
+
+        self.cg.module.cg_build_koji_tag = None
+        self.cg._tag_cg_build()
+
+        koji_session.tagBuild.assert_not_called()
+
+    @patch("module_build_service.builder.KojiModuleBuilder.get_session")
+    def test_tag_cg_build_no_tag_available(self, get_session):
+        """ Test that the CG build is not tagged when no tag available. """
+        koji_session = MagicMock()
+        koji_session.getTag.side_effect = [{}, {}]
+        get_session.return_value = koji_session
+
+        self.cg._tag_cg_build()
+
+        koji_session.tagBuild.assert_not_called()

tests/test_pdc.py

@@ -86,7 +86,7 @@ class TestPDCModule(unittest.TestCase):
             'version': 'master',
             'release': '20170315134803',
         }
-        result = mbs_pdc.get_module_build_dependencies(self.pdc, query)
+        result = mbs_pdc.get_module_build_dependencies(self.pdc, query).keys()
         expected = [
             u'module-bootstrap-rawhide',
         ]
@@ -103,7 +103,7 @@ class TestPDCModule(unittest.TestCase):
             'version': 'master',
             'release': '20170322155247'
         }
-        result = mbs_pdc.get_module_build_dependencies(self.pdc, query)
+        result = mbs_pdc.get_module_build_dependencies(self.pdc, query).keys()
         expected = [
             u'module-base-runtime-master-20170315134803',
         ]
@@ -127,7 +127,7 @@ class TestPDCModule(unittest.TestCase):
             
             build = module_build_service.models.ModuleBuild.local_modules(
                 db.session, "child", "master")
-            result = mbs_pdc.get_module_build_dependencies(self.pdc, build[0].mmd())
+            result = mbs_pdc.get_module_build_dependencies(self.pdc, build[0].mmd()).keys()
 
             local_path = os.path.join(base_dir, 'staged_data', "local_builds")
 

tests/test_scheduler/test_module_wait.py

@@ -31,7 +31,7 @@ import vcr
 import koji
 from tests import conf, db, app, scheduler_init_data
 from module_build_service import build_logs
-from module_build_service.models import ComponentBuild
+from module_build_service.models import ComponentBuild, ModuleBuild
 
 base_dir = os.path.dirname(os.path.dirname(__file__))
 cassette_dir = base_dir + '/vcr-request-data/'
@@ -154,3 +154,81 @@ class TestModuleWait(unittest.TestCase):
             module_build_service.scheduler.handlers.modules.wait(
                 config=conf, session=db.session, msg=msg)
             self.assertTrue(koji_session.newRepo.called)
+
+    @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups",
+           return_value={'build': [], 'srpm-build': []})
+    @patch("module_build_service.builder.KojiModuleBuilder.get_session")
+    @patch("module_build_service.builder.GenericBuilder.create_from_module")
+    @patch('module_build_service.pdc')
+    def test_set_cg_build_koji_tag_fallback_to_default(
+            self, pdc, create_builder, koji_get_session, dbg):
+        """
+        Test that build.cg_build_koji_tag fallbacks to default tag.
+        """
+        with app.app_context():
+            pdc.get_module_tag.return_value = "module-testmodule-master-20170109091357"
+            base_mmd = _modulemd.ModuleMetadata()
+            base_mmd.name = "base-runtime"
+            base_mmd.stream = "f27"
+            pdc.get_module_build_dependencies.return_value = {
+                "module-bootstrap-tag": base_mmd}
+
+            scheduler_init_data()
+            koji_session = mock.MagicMock()
+            koji_session.newRepo.return_value = 123456
+            koji_get_session.return_value = koji_session
+
+            builder = mock.MagicMock()
+            builder.koji_session = koji_session
+            builder.module_build_tag = {"name": "module-123-build"}
+            builder.get_disttag_srpm.return_value = 'some srpm disttag'
+            builder.build.return_value = 1234, koji.BUILD_STATES['BUILDING'], "", "module-build-macros-1-1"
+            create_builder.return_value = builder
+
+            msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=1,
+                                                           module_build_state='some state')
+            module_build_service.scheduler.handlers.modules.wait(
+                config=conf, session=db.session, msg=msg)
+            module_build = ModuleBuild.query.filter_by(id=1).one()
+            self.assertEqual(module_build.cg_build_koji_tag, "modular-updates-candidate")
+
+
+    @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups",
+           return_value={'build': [], 'srpm-build': []})
+    @patch("module_build_service.builder.KojiModuleBuilder.get_session")
+    @patch("module_build_service.builder.GenericBuilder.create_from_module")
+    @patch('module_build_service.pdc')
+    @patch("module_build_service.config.Config.base_module_names",
+           new_callable=mock.PropertyMock, return_value=set(["base-runtime"]))
+    def test_set_cg_build_koji_tag(
+            self, cfg, pdc, create_builder, koji_get_session, dbg):
+        """
+        Test that build.cg_build_koji_tag is set.
+        """
+        with app.app_context():
+            pdc.get_module_tag.return_value = "module-testmodule-master-20170109091357"
+            base_mmd = _modulemd.ModuleMetadata()
+            base_mmd.name = "base-runtime"
+            base_mmd.stream = "f27"
+            pdc.get_module_build_dependencies.return_value = {
+                "module-bootstrap-tag": base_mmd}
+
+            scheduler_init_data()
+            koji_session = mock.MagicMock()
+            koji_session.newRepo.return_value = 123456
+            koji_get_session.return_value = koji_session
+
+            builder = mock.MagicMock()
+            builder.koji_session = koji_session
+            builder.module_build_tag = {"name": "module-123-build"}
+            builder.get_disttag_srpm.return_value = 'some srpm disttag'
+            builder.build.return_value = 1234, koji.BUILD_STATES['BUILDING'], "", "module-build-macros-1-1"
+            create_builder.return_value = builder
+
+            msg = module_build_service.messaging.MBSModule(msg_id=None, module_build_id=1,
+                                                           module_build_state='some state')
+            module_build_service.scheduler.handlers.modules.wait(
+                config=conf, session=db.session, msg=msg)
+            module_build = ModuleBuild.query.filter_by(id=1).one()
+            self.assertEqual(module_build.cg_build_koji_tag,
+                             "f27-modular-updates-candidate")