Fix #276 - Use PDC to resolve buildroot components instead of MBS database

module_build_service/builder.py

@@ -52,6 +52,7 @@ from OpenSSL.SSL import SysCallError
 
 from module_build_service import conf, log, db
 from module_build_service.models import ModuleBuild
+from module_build_service import pdc
 import module_build_service.scm
 import module_build_service.utils
 import module_build_service.scheduler
@@ -277,6 +278,24 @@ class GenericBuilder(six.with_metaclass(ABCMeta)):
         """
         raise NotImplementedError()
 
+    @classmethod
+    def default_buildroot_groups(cls, session, module):
+        try:
+            pdc_session = pdc.get_pdc_client_session(conf)
+            pdc_groups = pdc.resolve_profiles(pdc_session, module.mmd(),
+                                            ('buildroot', 'srpm-buildroot'))
+            groups = {
+                'build': pdc_groups['buildroot'],
+                'srpm-build': pdc_groups['srpm-buildroot'],
+            }
+        except ValueError:
+            reason = "Failed to gather buildroot groups from SCM."
+            log.exception(reason)
+            module.transition(conf, state="failed", state_reason=reason)
+            session.commit()
+            raise
+        return groups
+
 class KojiModuleBuilder(GenericBuilder):
     """ Koji specific builder class """
 

module_build_service/pdc.py

@@ -183,6 +183,73 @@ def get_module_tag(session, module_info, strict=False):
     """
     return get_module(session, module_info, strict=strict)['koji_tag']
 
+def get_module_modulemd(session, module_info, strict=False):
+    """
+    :param session : PDCClient instance
+    :param module_info: list of module_info dicts
+    :param strict: Normally this function returns None if no module can be
+           found.  If strict=True, then a ValueError is raised.
+    :return: ModuleMetadata instance
+    """
+    yaml = get_module(session, module_info, strict=strict)['modulemd']
+    if not yaml:
+        if strict:
+            raise ValueError("Failed to find modulemd entry in PDC for "
+                "%r" % module_info)
+        else:
+            return None
+
+    mmd = modulemd.ModuleMetadata()
+    mmd.loads(yaml)
+    return mmd
+
+def resolve_profiles(session, mmd, keys, seen=None):
+    """
+    :param session : PDCClient instance
+    :param mmd: ModuleMetadata instance of module
+    :param keys: list of modulemd installation profiles to include in
+                 the result.
+    :return: Dictionary with keys set according to `keys` param and values
+             set to union of all components defined in all installation
+             profiles matching the key recursively using the buildrequires.
+
+    https://pagure.io/fm-orchestrator/issue/181
+    """
+
+    seen = seen or []  # Initialize to an empty list.
+    results = {}
+    for key in keys:
+        results[key] = set()
+    for name, stream in mmd.buildrequires.items():
+        # First, guard against infinite recursion
+        if name in seen:
+            continue
+
+        # Find the latest of the dep in our db of built modules.
+        module_info = {}
+        module_info['variant_id'] = name
+        module_info['variant_stream'] = stream
+        dep_mmd = get_module_modulemd(session, module_info, False)
+        # XXX - We may want to make this fatal one day, but warn for now.
+        if not dep_mmd:
+            log.warn("Could not find built dep "
+                        "%s/%s for %r" % (name, stream, mmd.name))
+            continue
+
+        # Take note of what rpms are in this dep's profile.
+        profiles = dep_mmd.profiles
+        for key in keys:
+            if key in profiles:
+                results[key] |= profiles[key].rpms
+
+        # And recurse to all modules that are deps of our dep.
+        rec_results = resolve_profiles(session, dep_mmd, keys, seen + [name])
+        for rec_key, rec_result in rec_results.items():
+            results[rec_key] |= rec_result
+
+    # Return the union of all rpms in all profiles of the given keys.
+    return results
+
 def module_depsolving_wrapper(session, module_list, strict=True):
     """
     :param session : PDCClient instance

module_build_service/scheduler/handlers/components.py

@@ -77,18 +77,8 @@ def _finalize(config, session, msg, state):
     builder = module_build_service.builder.GenericBuilder.create(
         parent.owner, module_name, config.system, config, tag_name=tag)
 
-    try:
-        groups = {
-            'build': parent.resolve_profiles(session, 'buildroot'),
-            'srpm-build': parent.resolve_profiles(session, 'srpm-buildroot'),
-        }
-    except ValueError:
-        reason = "Failed to gather buildroot groups from SCM."
-        log.exception(reason)
-        parent.transition(config, state=models.BUILD_STATES["failed"],
-                            state_reason=reason)
-        session.commit()
-        raise
+    groups = module_build_service.builder.GenericBuilder.default_buildroot_groups(
+        session, parent)
 
     builder.buildroot_connect(groups)
 

module_build_service/scheduler/handlers/modules.py

@@ -68,17 +68,8 @@ def failed(config, session, msg):
             and c.state != koji.BUILD_STATES["FAILED"])
     ]
 
-    try:
-        groups = {
-            'build': build.resolve_profiles(session, 'buildroot'),
-            'srpm-build': build.resolve_profiles(session, 'srpm-buildroot'),
-        }
-    except ValueError:
-        reason = "Failed to gather buildroot groups from SCM."
-        log.exception(reason)
-        build.transition(config, state="failed", state_reason=reason)
-        session.commit()
-        raise
+    groups = module_build_service.builder.GenericBuilder.default_buildroot_groups(
+        session, build)
 
     if build.koji_tag:
         builder = module_build_service.builder.GenericBuilder.create(
@@ -201,17 +192,8 @@ def wait(config, session, msg):
             session.commit()
             raise
 
-    try:
-        groups = {
-            'build': build.resolve_profiles(session, 'buildroot'),
-            'srpm-build': build.resolve_profiles(session, 'srpm-buildroot'),
-        }
-    except ValueError:
-        reason = "Failed to gather buildroot groups from SCM."
-        log.exception(reason)
-        build.transition(config, state="failed", state_reason=reason)
-        session.commit()
-        raise
+    groups = module_build_service.builder.GenericBuilder.default_buildroot_groups(
+        session, build)
 
     log.debug("Found tag=%s for module %r" % (tag, build))
     # Hang on to this information for later.  We need to know which build is

module_build_service/scheduler/handlers/repos.py

@@ -79,17 +79,8 @@ def done(config, session, msg):
         log.warn("Odd!  All components in batch failed for %r." % module_build)
         return
 
-    try:
-        groups = {
-            'build': module_build.resolve_profiles(session, 'buildroot'),
-            'srpm-build': module_build.resolve_profiles(session, 'srpm-buildroot'),
-        }
-    except ValueError:
-        reason = "Failed to gather buildroot groups from SCM."
-        log.exception(reason)
-        module_build.transition(config, state="failed", state_reason=reason)
-        session.commit()
-        raise
+    groups = module_build_service.builder.GenericBuilder.default_buildroot_groups(
+        session, module_build)
 
     builder = module_build_service.builder.GenericBuilder.create(
         module_build.owner, module_build.name, config.system, config,

tests/__init__.py

@@ -28,10 +28,34 @@ from module_build_service import db
 from module_build_service.config import init_config
 from module_build_service.models import ModuleBuild, ComponentBuild
 from module_build_service.utils import insert_fake_baseruntime
+import module_build_service.pdc
 
 app = module_build_service.app
 conf = init_config(app)
 
+def local_resolve_profiles(pdc_session, mmd, keys):
+    """
+    Returns the default srpm-buildroot and "buildroot" groups as they
+    would have been return from PDC.
+
+    We are not using real PDC for resolve_profiles in tests,
+    because it makes tests slower and depending on the remote dev
+    PDC instance which is not ideal.
+    """
+    default_groups = {
+        'srpm-buildroot':
+            set(['shadow-utils', 'fedora-release', 'redhat-rpm-config',
+                    'rpm-build', 'fedpkg-minimal', 'gnupg2', 'bash']),
+        'buildroot':
+            set(['unzip', 'fedora-release', 'tar', 'cpio', 'gawk',
+                    'gcc', 'xz', 'sed', 'findutils', 'util-linux', 'bash',
+                    'info', 'bzip2', 'grep', 'redhat-rpm-config',
+                    'diffutils', 'make', 'patch', 'shadow-utils',
+                    'coreutils', 'which', 'rpm-build', 'gzip', 'gcc-c++'])}
+
+    return default_groups
+
+module_build_service.pdc.resolve_profiles = local_resolve_profiles
 
 def init_data():
     db.session.remove()

tests/test_build/test_build.py

@@ -96,7 +96,18 @@ class TestModuleBuilder(GenericBuilder):
         TestModuleBuilder.on_tag_artifacts_cb = None
 
     def buildroot_connect(self, groups):
-        pass
+        default_groups = {
+            'srpm-build':
+                set(['shadow-utils', 'fedora-release', 'redhat-rpm-config',
+                     'rpm-build', 'fedpkg-minimal', 'gnupg2', 'bash']),
+            'build':
+                set(['unzip', 'fedora-release', 'tar', 'cpio', 'gawk',
+                     'gcc', 'xz', 'sed', 'findutils', 'util-linux', 'bash',
+                     'info', 'bzip2', 'grep', 'redhat-rpm-config',
+                     'diffutils', 'make', 'patch', 'shadow-utils',
+                     'coreutils', 'which', 'rpm-build', 'gzip', 'gcc-c++'])}
+        if groups != default_groups:
+            raise ValueError("Wrong groups in TestModuleBuilder.buildroot_connect()")
 
     def buildroot_prep(self):
         pass