Add the ability to override a buildrequired module stream based on a module's branch

This moves the functionality that was in rpkg to MBS. See the following
PRs for more context:
https://pagure.io/rpkg/pull-request/390
https://pagure.io/rpkg/pull-request/420

module_build_service/config.py

@@ -499,7 +499,23 @@ class Config(object):
         'allowed_users': {
             'type': set,
             'default': set(),
-            'desc': 'The users/service accounts that don\'t require to be part of a group'}
+            'desc': 'The users/service accounts that don\'t require to be part of a group'},
+        'br_stream_override_module': {
+            'type': str,
+            'default': 'platform',
+            'desc': ('The module name to override in the buildrequires based on the branch name. '
+                     '"br_stream_override_regexes" must also be set for this to take '
+                     'effect.')
+        },
+        'br_stream_override_regexes': {
+            'type': list,
+            'default': [],
+            'desc': ('The list of regexes used to parse the stream override from the branch name. '
+                     '"br_stream_override_module" must also be set for this to take '
+                     'effect. The regexes can contain multiple capture groups that will be '
+                     'concatenated. Any null capture groups will be ignored. The first regex that '
+                     'matches the branch will be used.')
+        },
     }
 
     def __init__(self, conf_section_obj):

module_build_service/utils/submit.py

@@ -30,6 +30,7 @@ import tempfile
 import os
 from multiprocessing.dummy import Pool as ThreadPool
 from datetime import datetime
+import copy
 from module_build_service.utils import to_text_type
 
 import kobo.rpmlib
@@ -516,10 +517,46 @@ def _apply_dep_overrides(mmd, params):
     :raises ValidationError: if one of the overrides doesn't apply
     """
     dep_overrides = {
-        'buildrequires': params.get('buildrequire_overrides', {}),
-        'requires': params.get('require_overrides', {})
+        'buildrequires': copy.copy(params.get('buildrequire_overrides', {})),
+        'requires': copy.copy(params.get('require_overrides', {}))
     }
 
+    # Parse the module's branch to determine if it should override the stream of the buildrequired
+    # module defined in conf.br_stream_override_module
+    branch_search = None
+    if params.get('branch') and conf.br_stream_override_module and conf.br_stream_override_regexes:
+        # Only parse the branch for a buildrequire override if the user didn't manually specify an
+        # override for the module specified in conf.br_stream_override_module
+        if not dep_overrides['buildrequires'].get(conf.br_stream_override_module):
+            branch_search = None
+            for regex in conf.br_stream_override_regexes:
+                branch_search = re.search(regex, params['branch'])
+                if branch_search:
+                    log.debug(
+                        'The stream override regex `%s` matched the branch %s',
+                        regex, params['branch'])
+                    break
+            else:
+                log.debug('No stream override regexes matched the branch "%s"', params['branch'])
+
+    # If a stream was parsed from the branch, then add it as a stream override for the module
+    # specified in conf.br_stream_override_module
+    if branch_search:
+        # Concatenate all the groups that are not None together to get the desired stream.
+        # This approach is taken in case there are sections to ignore.
+        # For instance, if we need to parse `el8.0.0` from `rhel-8.0.0`.
+        parsed_stream = ''.join(group for group in branch_search.groups() if group)
+        if parsed_stream:
+            dep_overrides['buildrequires'][conf.br_stream_override_module] = [parsed_stream]
+            log.info(
+                'The buildrequired stream of "%s" was overriden with "%s" based on the branch "%s"',
+                conf.br_stream_override_module, parsed_stream, params['branch'])
+        else:
+            log.warning(
+                ('The regex `%s` only matched empty capture groups on the branch "%s". The regex '
+                 'is invalid and should be rewritten.'),
+                regex, params['branch'])
+
     unused_dep_overrides = {
         'buildrequires': set(dep_overrides['buildrequires'].keys()),
         'requires': set(dep_overrides['requires'].keys())
@@ -541,6 +578,10 @@ def _apply_dep_overrides(mmd, params):
                 getattr(dep, 'set_' + dep_type)(reqs)
 
     for dep_type in unused_dep_overrides.keys():
+        # If a stream override was applied from parsing the branch and it wasn't applicable,
+        # just ignore it
+        if branch_search and conf.br_stream_override_module in unused_dep_overrides[dep_type]:
+            unused_dep_overrides[dep_type].remove(conf.br_stream_override_module)
         if unused_dep_overrides[dep_type]:
             raise ValidationError(
                 'The {} overrides for the following modules aren\'t applicable: {}'

tests/test_views/test_views.py

@@ -44,6 +44,7 @@ from module_build_service import db, version, Modulemd
 import module_build_service.config as mbs_config
 import module_build_service.scheduler.handlers.modules
 from module_build_service.utils import import_mmd, load_mmd
+from module_build_service.glib import dict_values, from_variant_dict
 
 
 user = ('Homer J. Simpson', set(['packager']))
@@ -1673,7 +1674,6 @@ class TestViews:
     def test_buildrequires_is_included_in_json_output(self):
         # Inject xmd/mbs/buildrequires into an existing module build for
         # assertion later.
-        from module_build_service.glib import dict_values, from_variant_dict
         from module_build_service.models import make_session
         from module_build_service import conf
         br_modulea = dict(stream='6', version='1', context='1234')
@@ -1903,3 +1903,96 @@ class TestViews:
         # this test is the same as the previous except YAML_SUBMIT_ALLOWED is False,
         # but it should still succeed since yaml is always allowed for scratch builds
         assert rv.status_code == 201
+
+    @pytest.mark.parametrize('branch, platform_override', (
+        ('10', None),
+        ('10-rhel-8.0.0', 'el8.0.0'),
+        ('10-LP-product1.2', 'product1.2'),
+    ))
+    @patch('module_build_service.auth.get_user', return_value=user)
+    @patch('module_build_service.scm.SCM')
+    @patch.object(module_build_service.config.Config, 'br_stream_override_regexes',
+                  new_callable=PropertyMock)
+    def test_submit_build_dep_override_from_branch(
+            self, mocked_regexes, mocked_scm, mocked_get_user, branch, platform_override):
+        """
+        Test that MBS will parse the SCM branch to determine the platform stream to buildrequire.
+        """
+        mocked_regexes.return_value = [
+            r'(?:rh)(el)(?:\-)(\d+\.\d+\.\d+)$',
+            r'(?:\-LP\-)(.+)$'
+        ]
+        init_data(data_size=1, multiple_stream_versions=True)
+        # Create a platform for whatever the override is so the build submission succeeds
+        if platform_override:
+            platform_mmd = load_mmd(path.join(base_dir, 'staged_data', 'platform.yaml'), True)
+            platform_mmd.set_stream(platform_override)
+            if platform_override == 'el8.0.0':
+                xmd = from_variant_dict(platform_mmd.get_xmd())
+                xmd['mbs']['virtual_streams'] = 'el8'
+                platform_mmd.set_xmd(dict_values(xmd))
+            import_mmd(db.session, platform_mmd)
+
+        FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
+                '620ec77321b2ea7b0d67d82992dda3e1d67055b4')
+
+        post_url = '/module-build-service/2/module-builds/'
+        scm_url = ('https://src.stg.fedoraproject.org/modules/testmodule.git?#68931c90de214d9d13fe'
+                   'efbd35246a81b6cb8d49')
+
+        rv = self.client.post(post_url, data=json.dumps({'branch': branch, 'scmurl': scm_url}))
+        data = json.loads(rv.data)
+        assert rv.status_code == 201
+
+        mmd = Modulemd.Module().new_from_string(data[0]['modulemd'])
+        assert len(mmd.get_dependencies()) == 1
+        dep = mmd.get_dependencies()[0]
+        if platform_override:
+            expected_br = {platform_override}
+        else:
+            expected_br = {'f28'}
+        assert set(dep.get_buildrequires()['platform'].get()) == expected_br
+        # The requires should not change
+        assert set(dep.get_requires()['platform'].get()) == {'f28'}
+
+    @patch('module_build_service.auth.get_user', return_value=user)
+    @patch('module_build_service.scm.SCM')
+    @patch.object(module_build_service.config.Config, 'br_stream_override_regexes',
+                  new_callable=PropertyMock)
+    def test_submit_build_dep_override_from_branch_br_override(
+            self, mocked_regexes, mocked_scm, mocked_get_user):
+        """
+        Test that when the branch includes a stream override for the platform module, that the
+        provided "buildrequire_override" for the platform module takes precedence.
+        """
+        mocked_regexes.return_value = [r'(?:\-LP\-)(.+)$']
+        init_data(data_size=1, multiple_stream_versions=True)
+        # Create a platform for the override so the build submission succeeds
+        platform_mmd = load_mmd(path.join(base_dir, 'staged_data', 'platform.yaml'), True)
+        platform_mmd.set_stream('product1.3')
+        import_mmd(db.session, platform_mmd)
+
+        FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
+                '620ec77321b2ea7b0d67d82992dda3e1d67055b4')
+
+        post_url = '/module-build-service/2/module-builds/'
+        scm_url = ('https://src.stg.fedoraproject.org/modules/testmodule.git?#68931c90de214d9d13fe'
+                   'efbd35246a81b6cb8d49')
+        json_input = {
+            'branch': '10-LP-product1.2',
+            'scmurl': scm_url,
+            'buildrequire_overrides': {'platform': ['product1.3']}
+        }
+
+        rv = self.client.post(post_url, data=json.dumps(json_input))
+        data = json.loads(rv.data)
+        assert rv.status_code == 201
+
+        mmd = Modulemd.Module().new_from_string(data[0]['modulemd'])
+        assert len(mmd.get_dependencies()) == 1
+        dep = mmd.get_dependencies()[0]
+        # The buildrequire_override value should take precedence over the stream override from
+        # parsing the branch
+        assert set(dep.get_buildrequires()['platform'].get()) == {'product1.3'}
+        # The requires should not change
+        assert set(dep.get_requires()['platform'].get()) == {'f28'}