Merge #647 Add 'requires' of 'build-required' modules to buildroot when building module.

module_build_service/builder/base.py

@@ -287,20 +287,7 @@ class GenericBuilder(six.with_metaclass(ABCMeta)):
             # Resolve default buildroot groups using the PDC, but only for
             # non-local modules.
             pdc_groups = pdc.resolve_profiles(pdc_session, mmd,
-                                              ('buildroot', 'srpm-buildroot'),
-                                              exclude=local_modules)
-
-            # For local modules, resolve the buildroot groups using local
-            # modulemd metadata.
-            for module_name, module_info in mmd.xmd['mbs']['buildrequires'].items():
-                key = module_name + "-" + module_info['stream']
-                if key in local_modules:
-                    local_build = local_modules[key]
-                    local_mmd = local_build.mmd()
-                    if 'buildroot' in local_mmd.profiles:
-                        pdc_groups['buildroot'] |= local_mmd.profiles['buildroot'].rpms
-                    if 'srpm-buildroot' in local_mmd.profiles:
-                        pdc_groups['srpm-buildroot'] |= local_mmd.profiles['srpm-buildroot'].rpms
+                                              ('buildroot', 'srpm-buildroot'))
 
             groups = {
                 'build': pdc_groups['buildroot'],

module_build_service/pdc.py

@@ -27,11 +27,14 @@
 
 import modulemd
 from pdc_client import PDCClient
+from module_build_service import db
+from module_build_service import models
 
 import inspect
 import pprint
 import logging
 import six
+import copy
 log = logging.getLogger()
 
 
@@ -78,9 +81,10 @@ def get_variant_dict(data):
         if not isinstance(data, dict):
             return False
 
-        for attr in ('variant_id', 'variant_stream'):
-            if attr not in data.keys():
-                return False
+        if ('variant_id' not in data or
+                ('variant_stream' not in data and
+                 'variant_version' not in data)):
+            return False
         return True
 
     def is_modulemd(data):
@@ -239,28 +243,114 @@ def _extract_modulemd(yaml, strict=False):
     return mmd
 
 
-def resolve_profiles(session, mmd, keys, exclude=None):
+def _get_recursively_required_modules(session, info, modules=None,
+                                      strict=False):
+    """
+    :param session: PDCClient instance
+    :param info: pdc variant_dict, str, mmd or module dict
+    :param modules: Used by recursion only, list of modules found by previous
+                    iteration of this method.
+    :param strict: Normally this function returns empty list if no module can
+                   be found.  If strict=True, then a ValueError is raised.
+
+    Returns list of modules by recursively querying PDC based on a "requires"
+    list of an input module represented by `info`. The returned list
+    therefore contains all modules the input module "requires".
+
+    If there are some modules loaded by utils.load_local_builds(...), these
+    local modules will be used instead of querying PDC for the particular
+    module found in local module builds database.
+
+    The returned list contains only "modulemd" and "koji_tag" fields returned
+    by PDC, because other fields are not known for local builds.
+    """
+    modules = modules or []
+
+    variant_dict = get_variant_dict(info)
+    local_modules = models.ModuleBuild.local_modules(
+        db.session, variant_dict["variant_id"],
+        variant_dict['variant_version'])
+    if local_modules:
+        local_module = local_modules[0]
+        log.info("Using local module %r as a dependency.",
+                 local_module)
+        mmd = local_module.mmd()
+        module_info = {}
+        module_info["modulemd"] = local_module.modulemd
+        module_info["koji_tag"] = local_module.koji_tag
+    else:
+        module_info = get_module(session, variant_dict, strict)
+        module_info = {k: v for k, v in module_info.items()
+                       if k in ["modulemd", "koji_tag"]}
+        module_info = {
+            'modulemd': module_info['modulemd'],
+            'koji_tag': module_info['koji_tag']
+        }
+
+        yaml = module_info['modulemd']
+        mmd = _extract_modulemd(yaml)
+
+    # Check if we have examined this koji_tag already - no need to do
+    # it again...
+    if module_info in modules:
+        return modules
+
+    modules.append(module_info)
+
+
+    # We want to use the same stream as the one used in the time this
+    # module was built. But we still should fallback to plain mmd.requires
+    # in case this module depends on some older module for which we did
+    # not populate mmd.xmd['mbs']['requires'].
+    if mmd.xmd.get('mbs') and mmd.xmd['mbs'].get('requires'):
+        requires = mmd.xmd['mbs']['requires']
+        requires = {name: data['stream'] for name, data in requires.items()}
+    else:
+        requires = mmd.requires
+
+    for name, stream in requires.items():
+        modified_dep = {
+            'name': name,
+            'version': stream,
+            # Only return details about module builds that finished
+            'active': True,
+        }
+        modules = _get_recursively_required_modules(
+            session, modified_dep, modules, strict)
+
+    return modules
+
+
+def resolve_profiles(session, mmd, keys):
     """
     :param session : PDCClient instance
     :param mmd: ModuleMetadata instance of module
     :param keys: list of modulemd installation profiles to include in
                  the result.
-    :param exclude: a set or map with the keys being $name-$stream
-                 to not look up in the PDC
     :return: Dictionary with keys set according to `keys` param and values
              set to union of all components defined in all installation
              profiles matching the key using the buildrequires.
 
+    If there are some modules loaded by utils.load_local_builds(...), these
+    local modules will be considered when returning the profiles.
+
     https://pagure.io/fm-orchestrator/issue/181
     """
 
-    exclude = exclude or []
-
     results = {}
     for key in keys:
         results[key] = set()
     for module_name, module_info in mmd.xmd['mbs']['buildrequires'].items():
-        if module_name + "-" + module_info['stream'] in exclude:
+        local_modules = models.ModuleBuild.local_modules(
+            db.session, module_name, module_info['stream'])
+        if local_modules:
+            local_module = local_modules[0]
+            log.info("Using local module %r to resolve profiles.",
+                        local_module)
+            dep_mmd = local_module.mmd()
+            for key in keys:
+                if key in dep_mmd.profiles:
+                    results[key] |= dep_mmd.profiles[key].rpms
             continue
 
         # Find the dep in the built modules in PDC
@@ -268,12 +358,17 @@ def resolve_profiles(session, mmd, keys, exclude=None):
             'variant_id': module_name,
             'variant_stream': module_info['stream'],
             'variant_release': module_info['version']}
-        dep_mmd = get_module_modulemd(session, module_info, True)
+        modules = _get_recursively_required_modules(
+            session, module_info, strict=True)
 
-        # Take note of what rpms are in this dep's profile.
-        for key in keys:
-            if key in dep_mmd.profiles:
-                results[key] |= dep_mmd.profiles[key].rpms
+        for module in modules:
+            yaml = module['modulemd']
+            dep_mmd = _extract_modulemd(yaml)
+
+            # Take note of what rpms are in this dep's profile.
+            for key in keys:
+                if key in dep_mmd.profiles:
+                    results[key] |= dep_mmd.profiles[key].rpms
 
     # Return the union of all rpms in all profiles of the given keys.
     return results
@@ -282,33 +377,37 @@ def resolve_profiles(session, mmd, keys, exclude=None):
 def get_module_build_dependencies(session, module_info, strict=False):
     """
     :param session : PDCClient instance
-    :param module_info : a dict containing filters for pdc
+    :param module_info : a dict containing filters for pdc or ModuleMetadata
+    instance.
     :param strict: Normally this function returns None if no module can be
            found.  If strict=True, then a ValueError is raised.
     :return final list of koji tags
 
-    Example minimal module_info {'variant_id': module_name, 'variant_version': module_version, 'variant_type': 'module'}
+    Example minimal module_info:
+        {
+            'variant_id': module_name,
+            'variant_version': module_version,
+            'variant_type': 'module'
+        }
     """
     log.debug("get_module_build_dependencies(%r, strict=%r)" % (module_info, strict))
     # XXX get definitive list of modules
 
-    queried_module = get_module(session, module_info, strict=strict)
-    yaml = queried_module['modulemd']
-    queried_mmd = _extract_modulemd(yaml, strict=strict)
-    if not queried_mmd or not queried_mmd.xmd.get('mbs') or not \
-            queried_mmd.xmd['mbs'].get('buildrequires'):
-        raise RuntimeError(
-            'The module "{0!r}" did not contain its modulemd or did not have '
-            'its xmd attribute filled out in PDC'.format(module_info))
-
     # This is the set we're going to build up and return.
     module_tags = set()
 
-    # Take note of the tag of this module, but only if it is a dep and
-    # not in the original list.
-    # XXX - But, for now go ahead and include it because that's how this
-    # code used to work.
-    module_tags.add(queried_module['koji_tag'])
+    if not isinstance(module_info, modulemd.ModuleMetadata):
+        queried_module = get_module(session, module_info, strict=strict)
+        yaml = queried_module['modulemd']
+        queried_mmd = _extract_modulemd(yaml, strict=strict)
+    else:
+        queried_mmd = module_info
+
+    if (not queried_mmd or not queried_mmd.xmd.get('mbs') or
+            not queried_mmd.xmd['mbs'].get('buildrequires')):
+        raise RuntimeError(
+            'The module "{0!r}" did not contain its modulemd or did not have '
+            'its xmd attribute filled out in PDC'.format(module_info))
 
     buildrequires = queried_mmd.xmd['mbs']['buildrequires']
     # Queue up the next tier of deps that we should look at..
@@ -320,8 +419,10 @@ def get_module_build_dependencies(session, module_info, strict=False):
             # Only return details about module builds that finished
             'active': True,
         }
-        info = get_module(session, modified_dep, strict)
-        module_tags.add(info['koji_tag'])
+        modules = _get_recursively_required_modules(
+            session, modified_dep, strict=strict)
+        tags = [m["koji_tag"] for m in modules]
+        module_tags = module_tags.union(set(tags))
 
     return module_tags
 
@@ -356,3 +457,59 @@ def get_module_commit_hash_and_version(session, module_info):
         log.warn(
             'The version for {0!r} was not in PDC'.format(module_info))
     return commit_hash, version
+
+def resolve_requires(session, requires):
+    """
+    Takes `requires` dict with module_name as key and module_stream as value.
+    Resolves the stream to particular latest version of a module and returns
+    new dict in following format:
+
+    {
+        "module_name": {
+            "ref": module_commit_hash,
+            "stream": original_module_stream,
+            "version": module_version,
+        },
+        ...
+    }
+
+    If there are some modules loaded by utils.load_local_builds(...), these
+    local modules will be considered when resolving the requires.
+
+    Raises RuntimeError on PDC lookup error.
+    """
+    new_requires = copy.deepcopy(requires)
+    for module_name, module_stream in requires.items():
+        # Try to find out module dependency in the local module builds
+        # added by utils.load_local_builds(...).
+        local_modules = models.ModuleBuild.local_modules(
+            db.session, module_name, module_stream)
+        if local_modules:
+            local_build = local_modules[0]
+            new_requires[module_name] = {
+                # The commit ID isn't currently saved in modules.yaml
+                'ref': None,
+                'stream': local_build.stream,
+                'version': local_build.version
+            }
+            continue
+
+        # Assumes that module_stream is the stream and not the commit hash
+        module_info = {
+            'name': module_name,
+            'version': module_stream,
+            'active': True}
+        commit_hash, version = get_module_commit_hash_and_version(
+            session, module_info)
+        if version and commit_hash:
+            new_requires[module_name] = {
+                'ref': commit_hash,
+                'stream': module_stream,
+                'version': str(version)
+            }
+        else:
+            raise RuntimeError(
+                'The module "{0}" didn\'t contain either a commit hash or a'
+                ' version in PDC'.format(module_name))
+
+    return new_requires

module_build_service/scheduler/handlers/modules.py

@@ -148,6 +148,7 @@ def wait(config, session, msg):
     build_logs.start(build.id)
 
     log.info("Found build=%r from message" % build)
+    log.info("%r", build.modulemd)
 
     module_info = build.json()
     if module_info['state'] != msg.module_build_state:
@@ -160,75 +161,55 @@ def wait(config, session, msg):
     tag = None
     dependencies = []
 
-    if conf.system != "koji":
-        # In case of mock, we do not try to get anything from pdc,
-        # just generate our own koji_tag to identify the module in messages.
-        tag = '-'.join(['module',
-                        module_info['name'],
-                        str(module_info['stream']), str(module_info['version'])])
+    pdc_session = module_build_service.pdc.get_pdc_client_session(config)
 
-        for name, stream in build.mmd().buildrequires.items():
-            # Try to load local module if it is loaded by
-            # utils.load_local_modules(...). Such modules have koji_tag set to
-            # path to repository with built RPMs.
-            local_modules = models.ModuleBuild.local_modules(session, name, stream)
-            if local_modules:
-                local_module = local_modules[0]
-                log.info("Using local module %r as a dependency.",
-                         local_module)
-                dependencies.append(local_module.koji_tag)
-                continue
+    @module_build_service.utils.retry(
+        interval=10, timeout=120,
+        wait_on=(ValueError, RuntimeError, ConnectionError))
+    def _get_deps_and_tag():
+        """
+        Private method to get the dependencies and koji tag of a module we
+        are going to build. We use private method here to allow "retry"
+        on failure.
+        """
+        if conf.system != "koji":
+            # In case of non-koji backend, we want to get the dependencies
+            # of the local module build based on ModuleMetadata, because the
+            # local build is not stored in PDC and therefore we cannot query
+            # it using the `pdc_query` as for Koji below.
+            dependencies = module_build_service.pdc.get_module_build_dependencies(
+                pdc_session, build.mmd(), strict=True)
 
-            pdc_session = module_build_service.pdc.get_pdc_client_session(config)
+            # We also don't want to get the tag name from the PDC, but just
+            # generate it locally instead.
+            tag = '-'.join(['module',
+                            module_info['name'],
+                            str(module_info['stream']), str(module_info['version'])])
+        else:
+            # For Koji backend, query the PDC for the module we are going to
+            # build to get the koji_tag and deps from it.
             pdc_query = {
-                'name': name,
-                'version': stream,
-                'active': True
+                'name': module_info['name'],
+                'version': module_info['stream'],
+                'release': module_info['version'],
             }
-
-            @module_build_service.utils.retry(interval=10, timeout=30, wait_on=ValueError)
-            def _get_module():
-                log.info("Getting %s from pdc (query %r)" % (module_info['name'], pdc_query))
-                return module_build_service.pdc.get_module_tag(
-                    pdc_session, pdc_query, strict=True)
-
-            try:
-                dependencies.append(_get_module())
-            except ValueError:
-                reason = "Failed to get module info from PDC. Max retries reached."
-                log.exception(reason)
-                build.transition(config, state="failed", state_reason=reason)
-                session.commit()
-                raise
-    else:
-        # TODO: Move this to separate func
-        pdc_session = module_build_service.pdc.get_pdc_client_session(config)
-        pdc_query = {
-            'name': module_info['name'],
-            'version': module_info['stream'],
-            'release': module_info['version'],
-        }
-
-        @module_build_service.utils.retry(
-            interval=10, timeout=120,
-            wait_on=(ValueError, RuntimeError, ConnectionError))
-        def _get_deps_and_tag():
             log.info("Getting %s deps from pdc (query %r)" % (module_info['name'], pdc_query))
             dependencies = module_build_service.pdc.get_module_build_dependencies(
                 pdc_session, pdc_query, strict=True)
             log.info("Getting %s tag from pdc (query %r)" % (module_info['name'], pdc_query))
             tag = module_build_service.pdc.get_module_tag(
                 pdc_session, pdc_query, strict=True)
-            return dependencies, tag
 
-        try:
-            dependencies, tag = _get_deps_and_tag()
-        except ValueError:
-            reason = "Failed to get module info from PDC. Max retries reached."
-            log.exception(reason)
-            build.transition(config, state="failed", state_reason=reason)
-            session.commit()
-            raise
+        return dependencies, tag
+
+    try:
+        dependencies, tag = _get_deps_and_tag()
+    except ValueError:
+        reason = "Failed to get module info from PDC. Max retries reached."
+        log.exception(reason)
+        build.transition(config, state="failed", state_reason=reason)
+        session.commit()
+        raise
 
     log.debug("Found tag=%s for module %r" % (tag, build))
     # Hang on to this information for later.  We need to know which build is

module_build_service/utils.py

@@ -47,7 +47,7 @@ from module_build_service.errors import (Forbidden, Conflict)
 import module_build_service.messaging
 from multiprocessing.dummy import Pool as ThreadPool
 import module_build_service.pdc
-from module_build_service.pdc import get_module_commit_hash_and_version
+from module_build_service.pdc import resolve_requires
 
 import concurrent.futures
 
@@ -690,47 +690,21 @@ def format_mmd(mmd, scmurl):
         else:
             mmd.xmd['mbs']['commit'] = scm.get_latest()
 
-    # If the modulemd yaml specifies module buildrequires, replace the streams
-    # with commit hashes
+    pdc = module_build_service.pdc.get_pdc_client_session(conf)
+
+    # Resolve Build-requires.
     if mmd.buildrequires:
-        mmd.xmd['mbs']['buildrequires'] = copy.deepcopy(mmd.buildrequires)
-        pdc = module_build_service.pdc.get_pdc_client_session(conf)
-        for module_name, module_stream in \
-                mmd.xmd['mbs']['buildrequires'].items():
-
-            # Try to find out module dependency in the local module builds
-            # added by utils.load_local_builds(...).
-            local_modules = models.ModuleBuild.local_modules(
-                db.session, module_name, module_stream)
-            if local_modules:
-                local_build = local_modules[0]
-                mmd.xmd['mbs']['buildrequires'][module_name] = {
-                    # The commit ID isn't currently saved in modules.yaml
-                    'stream': local_build.stream,
-                    'version': local_build.version
-                }
-                continue
-
-            # Assumes that module_stream is the stream and not the commit hash
-            module_info = {
-                'name': module_name,
-                'version': module_stream,
-                'active': True}
-            commit_hash, version = get_module_commit_hash_and_version(
-                pdc, module_info)
-            if version and (commit_hash or not scmurl):
-                mmd.xmd['mbs']['buildrequires'][module_name] = {
-                    'ref': commit_hash,
-                    'stream': mmd.buildrequires[module_name],
-                    'version': str(version)
-                }
-            else:
-                raise RuntimeError(
-                    'The module "{0}" didn\'t contain either a commit hash or a'
-                    ' version in PDC'.format(module_name))
+        mmd.xmd['mbs']['buildrequires'] = resolve_requires(
+            pdc, mmd.buildrequires)
     else:
         mmd.xmd['mbs']['buildrequires'] = {}
 
+    # Resolve Requires.
+    if mmd.requires:
+        mmd.xmd['mbs']['requires'] = resolve_requires(pdc, mmd.requires)
+    else:
+        mmd.xmd['mbs']['requires'] = {}
+
     if mmd.components:
         if 'rpms' not in mmd.xmd['mbs']:
             mmd.xmd['mbs']['rpms'] = {}

tests/staged_data/local_builds/module-child-master-20170816080815/results/modules.yaml

@@ -0,0 +1,38 @@
+---
+data:
+  api:
+    rpms: [ed, mksh]
+  artifacts:
+    rpms: [ed-debuginfo-1.14.1-4.module_92fc48de.x86_64, ed-1.14.1-4.module_92fc48de.x86_64,
+      module-build-macros-0.1-1.module_92fc48de.src, module-build-macros-0.1-1.module_92fc48de.noarch,
+      ed-1.14.1-4.module_92fc48de.src]
+  components:
+    rpms:
+      ed: {cache: 'http://pkgs.fedoraproject.org/repo/pkgs/ed', rationale: A build
+          dependency of mksh., ref: master, repository: 'git://pkgs.fedoraproject.org/rpms/ed'}
+  dependencies:
+    buildrequires: {parent: master}
+    requires: {parent: master}
+  description: This module demonstrates how to write simple modulemd files And can
+    be used for testing the build and release pipeline.
+  license:
+    module: [MIT]
+  name: child
+  profiles:
+    default:
+      rpms: [mksh]
+  references: {community: 'https://docs.pagure.org/modularity/', documentation: 'https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules'}
+  stream: master
+  summary: A test module in all its beautiful beauty
+  version: 20170816080815
+  xmd:
+    mbs:
+      buildrequires:
+        parent: {ref: 147dca4ca65aa9a1ac51f71b7e687f9178ffa5df, stream: master,
+          version: '20170616125652'}
+      commit: 722fd739fd6cf66faf29f6fb95dd64f60ba3e39a
+      rpms:
+        ed: {ref: 01bf8330812fea798671925cc537f2f29b0bd216}
+      scmurl: file:///home/hanzz/modules/testmodule/
+document: modulemd
+version: 1

tests/staged_data/local_builds/module-parent-master-20170816080815/results/modules.yaml

@@ -0,0 +1,41 @@
+---
+data:
+  api:
+    rpms: [ed, mksh]
+  artifacts:
+    rpms: [ed-debuginfo-1.14.1-4.module_92fc48de.x86_64, ed-1.14.1-4.module_92fc48de.x86_64,
+      module-build-macros-0.1-1.module_92fc48de.src, module-build-macros-0.1-1.module_92fc48de.noarch,
+      ed-1.14.1-4.module_92fc48de.src]
+  components:
+    rpms:
+      ed: {cache: 'http://pkgs.fedoraproject.org/repo/pkgs/ed', rationale: A build
+          dependency of mksh., ref: master, repository: 'git://pkgs.fedoraproject.org/rpms/ed'}
+  dependencies:
+    buildrequires: {testmodule: master}
+    requires: {base-runtime: master}
+  description: This module demonstrates how to write simple modulemd files And can
+    be used for testing the build and release pipeline.
+  license:
+    module: [MIT]
+  name: parent
+  profiles:
+    default:
+      rpms: [mksh]
+  references: {community: 'https://docs.pagure.org/modularity/', documentation: 'https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules'}
+  stream: master
+  summary: A test module in all its beautiful beauty
+  version: 20170816080815
+  xmd:
+    mbs:
+      buildrequires:
+        testmodule: {ref: 147dca4ca65aa9a1ac51f71b7e687f9178ffa5df, stream: master,
+          version: '20170616125652'}
+      requires:
+        base-runtime: {ref: 147dca4ca65aa9a1ac51f71b7e687f9178ffa5df, stream: master,
+          version: '20170616125652'}
+      commit: 722fd739fd6cf66faf29f6fb95dd64f60ba3e39a
+      rpms:
+        ed: {ref: 01bf8330812fea798671925cc537f2f29b0bd216}
+      scmurl: file:///home/hanzz/modules/testmodule/
+document: modulemd
+version: 1

tests/test_pdc.py

@@ -23,9 +23,13 @@
 import os
 
 import unittest
+from mock import patch, PropertyMock
 
 import vcr
 import module_build_service.pdc as mbs_pdc
+import module_build_service.utils
+import module_build_service.models
+from module_build_service import app, db
 
 import tests
 import modulemd
@@ -67,9 +71,6 @@ class TestPDCModule(unittest.TestCase):
         result = mbs_pdc.get_module_build_dependencies(self.pdc, query)
         expected = [
             u'module-bootstrap-rawhide',
-            # Should the list of deps should not include the original tag?
-            # Probably not.
-            u'module-base-runtime-master-20170315134803',
         ]
         self.assertEqual(set(result), set(expected))
 
@@ -87,12 +88,41 @@ class TestPDCModule(unittest.TestCase):
         result = mbs_pdc.get_module_build_dependencies(self.pdc, query)
         expected = [
             u'module-base-runtime-master-20170315134803',
-            # Should the list of deps should not include the original tag?
-            # Probably not.
-            u'module-testmodule-master-20170322155247'
         ]
         self.assertEqual(set(result), set(expected))
 
+    @patch("module_build_service.config.Config.system",
+        new_callable=PropertyMock, return_value="test")
+    @patch("module_build_service.config.Config.mock_resultsdir",
+        new_callable=PropertyMock,
+        return_value=os.path.join(
+            base_dir, 'staged_data', "local_builds"))
+    def test_get_module_build_dependencies_recursive_requires(
+            self, resultdir, conf_system):
+        """
+        Tests that we return Requires of Buildrequires of a module
+        recursively.
+        """
+        with app.app_context():
+            module_build_service.utils.load_local_builds(
+                ["base-runtime", "parent", "child", "testmodule"])
+            
+            build = module_build_service.models.ModuleBuild.local_modules(
+                db.session, "child", "master")
+            result = mbs_pdc.get_module_build_dependencies(self.pdc, build[0].mmd())
+
+            local_path = os.path.join(base_dir, 'staged_data', "local_builds")
+
+            expected = [
+                os.path.join(
+                    local_path,
+                    'module-base-runtime-master-20170816080815/results'),
+                os.path.join(
+                    local_path,
+                    'module-parent-master-20170816080815/results'),
+            ]
+            self.assertEqual(set(result), set(expected))
+
     def test_resolve_profiles(self):
         current_dir = os.path.dirname(__file__)
         yaml_path = os.path.join(
@@ -115,19 +145,27 @@ class TestPDCModule(unittest.TestCase):
         }
         self.assertEqual(result, expected)
 
-    def test_resolve_profiles_exclude(self):
-        current_dir = os.path.dirname(__file__)
-        yaml_path = os.path.join(
-            current_dir, 'staged_data', 'formatted_testmodule.yaml')
-        mmd = modulemd.ModuleMetadata()
-        mmd.load(yaml_path)
-        result = mbs_pdc.resolve_profiles(self.pdc, mmd,
-                                          ('buildroot', 'srpm-buildroot'),
-                                          exclude=("base-runtime-master",))
-        expected = {
-            'buildroot':
-                set(),
-            'srpm-buildroot':
-                set()
-        }
-        self.assertEqual(result, expected)
+    @patch("module_build_service.config.Config.system",
+        new_callable=PropertyMock, return_value="test")
+    @patch("module_build_service.config.Config.mock_resultsdir",
+        new_callable=PropertyMock,
+        return_value=os.path.join(
+            base_dir, 'staged_data', "local_builds"))
+    def test_resolve_profiles_local_module(self, local_builds, conf_system):
+        with app.app_context():
+            module_build_service.utils.load_local_builds(["base-runtime"])
+
+            current_dir = os.path.dirname(__file__)
+            yaml_path = os.path.join(
+                current_dir, 'staged_data', 'formatted_testmodule.yaml')
+            mmd = modulemd.ModuleMetadata()
+            mmd.load(yaml_path)
+            result = mbs_pdc.resolve_profiles(self.pdc, mmd,
+                                            ('buildroot', 'srpm-buildroot'))
+            expected = {
+                'buildroot':
+                    set(['foo']),
+                'srpm-buildroot':
+                    set(['bar'])
+            }
+            self.assertEqual(result, expected)

tests/test_utils/test_utils.py

@@ -121,6 +121,11 @@ class TestUtils(unittest.TestCase):
                         'ref': '147dca4ca65aa9a1ac51f71b7e687f9178ffa5df',
                         'stream': 'master',
                         'version': '20170616125652'}},
+                'requires': {
+                    'base-runtime': {
+                        'version': '20170616125652',
+                        'ref': '147dca4ca65aa9a1ac51f71b7e687f9178ffa5df',
+                        'stream': 'master'}},
                 'rpms': {'perl-List-Compare': {'ref': '76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb'},
                          'perl-Tangerine': {'ref': '4ceea43add2366d8b8c5a622a2fb563b625b9abf'},
                          'tangerine': {'ref': 'fbed359411a1baa08d4a88e0d12d426fbf8f602c'}},
@@ -159,6 +164,11 @@ class TestUtils(unittest.TestCase):
                         'ref': '147dca4ca65aa9a1ac51f71b7e687f9178ffa5df',
                         'stream': 'master',
                         'version': '20170616125652'}},
+                'requires': {
+                    'base-runtime': {
+                        'version': '20170616125652',
+                        'ref': '147dca4ca65aa9a1ac51f71b7e687f9178ffa5df',
+                        'stream': 'master'}},
                 'rpms': {'perl-List-Compare': {'ref': '76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb'},
                          'perl-Tangerine': {'ref': '4ceea43add2366d8b8c5a622a2fb563b625b9abf'},
                          'tangerine': {'ref': 'fbed359411a1baa08d4a88e0d12d426fbf8f602c'}},

tests/vcr-request-data/tests.test_build.test_build.TestBuild.test_try_to_reach_concurrent_threshold

@@ -24,14 +24,14 @@ interactions:
         0nt+dDwu4vN84wX/fP38A7xxstJVBAAA
     headers:
       allow: ['GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS']
-      appserver: [proxy04.fedoraproject.org]
-      apptime: [D=283912]
+      appserver: [proxy08.fedoraproject.org]
+      apptime: [D=231339]
       cache-control: ['private, max-age=0, must-revalidate']
       connection: [Keep-Alive]
       content-encoding: [gzip]
       content-length: ['537']
       content-type: [application/json]
-      date: ['Thu, 07 Sep 2017 06:50:22 GMT']
+      date: ['Thu, 07 Sep 2017 09:44:55 GMT']
       keep-alive: ['timeout=15, max=500']
       server: [Apache/2.4.27 (Fedora) mod_wsgi/4.5.15 Python/2.7]
       set-cookie: [SERVERID=pdc-web02; path=/]
@@ -44,6 +44,7 @@ interactions:
     headers:
       Accept-Encoding: ['gzip, deflate']
       Connection: [keep-alive]
+      Cookie: [SERVERID=pdc-web02]
       User-Agent: [python-requests/2.13.0]
       accept: [application/json]
       content-type: [application/json]
@@ -64,14 +65,53 @@ interactions:
         0nt+dDwu4vN84wX/fP38A7xxstJVBAAA
     headers:
       allow: ['GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS']
-      appserver: [proxy11.fedoraproject.org]
-      apptime: [D=425383]
+      appserver: [proxy08.fedoraproject.org]
+      apptime: [D=275016]
       cache-control: ['private, max-age=0, must-revalidate']
       connection: [Keep-Alive]
       content-encoding: [gzip]
       content-length: ['537']
       content-type: [application/json]
-      date: ['Thu, 07 Sep 2017 06:50:24 GMT']
+      date: ['Thu, 07 Sep 2017 09:44:55 GMT']
+      keep-alive: ['timeout=15, max=499']
+      server: [Apache/2.4.27 (Fedora) mod_wsgi/4.5.15 Python/2.7]
+      strict-transport-security: [max-age=15768000; includeSubDomains; preload]
+      vary: ['Accept,Cookie,Accept-Encoding']
+      x-frame-options: [SAMEORIGIN]
+    status: {code: 200, message: OK}
+- request:
+    body: null
+    headers:
+      Accept-Encoding: ['gzip, deflate']
+      Connection: [keep-alive]
+      User-Agent: [python-requests/2.13.0]
+      accept: [application/json]
+      content-type: [application/json]
+    method: GET
+    uri: https://pdc.fedoraproject.org/rest_api/v1/unreleasedvariants/?variant_release=1&variant_version=master&page_size=1&variant_id=bootstrap
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAA41Uy27jMAz8FcKXHioHSdNNW9/2O9oioCU60cZ6rB5p2qD/vpTjJN5dFOhFEIfi
+        cIayfKykyzZVzUJUlg68sbnvReUD7bXL8RwHirlPHD4fqz0GjTattaqaqnUuxRTQV+KSyH9naoMx
+        UagXkyMWDX1Rnd59SRmnck8TfE8hamdLauCbpAL1hLFUlR4790uvE24uJPVVSsC3rVaF9pQyRanC
+        hM2LBVAUZdA+cZsGbmaz2U1Bi9YGLhwF8sF1uqc4VAG0Wfcq8IExBgjexAaeW4xbAe2H9ncCpAuU
+        k+4jb712ApTuuhHoSLmA9WiEY23VmNrg206ceQE2Uoqy1PL2ljeBPK/cQIC2HZMa3HG9xyS5cyC1
+        xVSzmlo62+mNKMrqQe+EM5ISELeo3Fs9dk0YBGQ7EBeo7rXNBwE8vkJ8+Hg9lccL39f+/zNHyu82
+        tdFWG+xZvs1+c/d9tROhgwq+FkLTwOm7GJBsDIb3Bn5Cx/O4Xh6crp09sWdIbpJJWyojDEwSskw5
+        0KxQjZ9dA4sSHYwaPZo2Xtye/NPvrANdUbiSN3AM1DVwv7pb0H33iN1SzXGF7Woul49qvljJH0/L
+        p/bhYTV/XC468Y+nifupos8zLJ0xOn2f/8UqJ7MhyzXnh/Bir8T8PgL/FrShtSJfnv2rqAaXkxhl
+        0nt+dDwu4vN84wX/fP38A7xxstJVBAAA
+    headers:
+      allow: ['GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS']
+      appserver: [proxy06.fedoraproject.org]
+      apptime: [D=250626]
+      cache-control: ['private, max-age=0, must-revalidate']
+      connection: [Keep-Alive]
+      content-encoding: [gzip]
+      content-length: ['537']
+      content-type: [application/json]
+      date: ['Thu, 07 Sep 2017 09:44:58 GMT']
       keep-alive: ['timeout=15, max=500']
       server: [Apache/2.4.27 (Fedora) mod_wsgi/4.5.15 Python/2.7]
       set-cookie: [SERVERID=pdc-web02; path=/]

tests/vcr-request-data/tests.test_pdc.TestPDCModule.test_get_module_build_dependencies

@@ -235,12 +235,12 @@ interactions:
     headers:
       allow: ['GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS']
       appserver: [proxy12.fedoraproject.org]
-      apptime: [D=396210]
+      apptime: [D=844637]
       cache-control: ['private, max-age=0, must-revalidate']
       connection: [Keep-Alive]
       content-encoding: [gzip]
       content-type: [application/json]
-      date: ['Thu, 07 Sep 2017 06:50:35 GMT']
+      date: ['Thu, 07 Sep 2017 10:38:35 GMT']
       keep-alive: ['timeout=15, max=500']
       server: [Apache/2.4.27 (Fedora) mod_wsgi/4.5.15 Python/2.7]
       set-cookie: [SERVERID=pdc-web02; path=/]
@@ -258,7 +258,7 @@ interactions:
       accept: [application/json]
       content-type: [application/json]
     method: GET
-    uri: https://pdc.fedoraproject.org/rest_api/v1/unreleasedvariants/?active=True&variant_release=1&variant_version=master&page_size=1&variant_id=bootstrap
+    uri: https://pdc.fedoraproject.org/rest_api/v1/unreleasedvariants/?variant_release=1&variant_version=master&page_size=1&variant_id=bootstrap
   response:
     body:
       string: !!binary |
@@ -275,13 +275,13 @@ interactions:
     headers:
       allow: ['GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS']
       appserver: [proxy12.fedoraproject.org]
-      apptime: [D=323630]
+      apptime: [D=712639]
       cache-control: ['private, max-age=0, must-revalidate']
       connection: [Keep-Alive]
       content-encoding: [gzip]
       content-length: ['537']
       content-type: [application/json]
-      date: ['Thu, 07 Sep 2017 06:50:37 GMT']
+      date: ['Thu, 07 Sep 2017 10:38:37 GMT']
       keep-alive: ['timeout=15, max=499']
       server: [Apache/2.4.27 (Fedora) mod_wsgi/4.5.15 Python/2.7]
       strict-transport-security: [max-age=15768000; includeSubDomains; preload]

tests/vcr-request-data/tests.test_pdc.TestPDCModule.test_get_module_build_dependencies_single_level

@@ -32,13 +32,13 @@ interactions:
     headers:
       allow: ['GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS']
       appserver: [proxy06.fedoraproject.org]
-      apptime: [D=199830]
+      apptime: [D=308440]
       cache-control: ['private, max-age=0, must-revalidate']
       connection: [Keep-Alive]
       content-encoding: [gzip]
       content-length: ['963']
       content-type: [application/json]
-      date: ['Thu, 07 Sep 2017 06:50:38 GMT']
+      date: ['Thu, 07 Sep 2017 10:38:39 GMT']
       keep-alive: ['timeout=15, max=500']
       server: [Apache/2.4.27 (Fedora) mod_wsgi/4.5.15 Python/2.7]
       set-cookie: [SERVERID=pdc-web02; path=/]
@@ -56,7 +56,7 @@ interactions:
       accept: [application/json]
       content-type: [application/json]
     method: GET
-    uri: https://pdc.fedoraproject.org/rest_api/v1/unreleasedvariants/?active=True&variant_release=20170315134803&variant_version=master&page_size=1&variant_id=base-runtime
+    uri: https://pdc.fedoraproject.org/rest_api/v1/unreleasedvariants/?variant_release=20170315134803&variant_version=master&page_size=1&variant_id=base-runtime
   response:
     body:
       string: !!binary |
@@ -283,12 +283,12 @@ interactions:
     headers:
       allow: ['GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS']
       appserver: [proxy06.fedoraproject.org]
-      apptime: [D=287940]
+      apptime: [D=368153]
       cache-control: ['private, max-age=0, must-revalidate']
       connection: [Keep-Alive]
       content-encoding: [gzip]
       content-type: [application/json]
-      date: ['Thu, 07 Sep 2017 06:50:39 GMT']
+      date: ['Thu, 07 Sep 2017 10:38:40 GMT']
       keep-alive: ['timeout=15, max=499']
       server: [Apache/2.4.27 (Fedora) mod_wsgi/4.5.15 Python/2.7]
       strict-transport-security: [max-age=15768000; includeSubDomains; preload]