fm-orchestrator/tests/test_build/test_build.py

# Copyright (c) 2016  Red Hat, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Written by Jan Kaluza <jkaluza@redhat.com>

import koji
import os
import re
from os import path, mkdir
from os.path import dirname
from shutil import copyfile
from datetime import datetime, timedelta
from random import randint
from module_build_service.utils import to_text_type

import module_build_service.messaging
import module_build_service.scheduler.handlers.repos
import module_build_service.utils
from module_build_service.errors import Forbidden
from module_build_service import db, models, conf, build_logs

from mock import patch, PropertyMock, Mock
from werkzeug.datastructures import FileStorage
import kobo
import pytest

from tests import app, reuse_component_init_data, clean_database
import json
import itertools

from module_build_service.builder.base import GenericBuilder
from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder
import module_build_service.scheduler.consumer
from module_build_service.messaging import MBSModule

base_dir = dirname(dirname(__file__))

user = ('Homer J. Simpson', set(['packager']))


class FakeSCM(object):
    def __init__(self, mocked_scm, name, mmd_filename, commit=None, version=20180205135154):
        self.mocked_scm = mocked_scm
        self.name = name
        self.commit = commit
        self.version = version
        self.mmd_filename = mmd_filename
        self.sourcedir = None

        self.mocked_scm.return_value.checkout = self.checkout
        self.mocked_scm.return_value.name = self.name
        self.mocked_scm.return_value.branch = 'master'
        self.mocked_scm.return_value.get_latest = self.get_latest
        self.mocked_scm.return_value.commit = self.commit
        self.mocked_scm.return_value.version = self.version
        self.mocked_scm.return_value.repository_root = "https://src.stg.fedoraproject.org/modules/"
        self.mocked_scm.return_value.sourcedir = self.sourcedir
        self.mocked_scm.return_value.get_module_yaml = self.get_module_yaml

    def checkout(self, temp_dir):
        self.sourcedir = path.join(temp_dir, self.name)
        mkdir(self.sourcedir)
        base_dir = path.abspath(path.dirname(__file__))
        copyfile(path.join(base_dir, '..', 'staged_data', self.mmd_filename),
                 self.get_module_yaml())

        return self.sourcedir

    def get_latest(self, ref='master'):
        return ref

    def get_module_yaml(self):
        return path.join(self.sourcedir, self.name + ".yaml")


class FakeModuleBuilder(GenericBuilder):
    """
    Fake module builder which succeeds for every build.
    """

    backend = "test"
    BUILD_STATE = "COMPLETE"
    # Simulates a situation when a component is already built in Koji
    INSTANT_COMPLETE = False
    DEFAULT_GROUPS = None

    on_build_cb = None
    on_cancel_cb = None
    on_finalize_cb = None
    on_buildroot_add_artifacts_cb = None
    on_tag_artifacts_cb = None

    @module_build_service.utils.validate_koji_tag('tag_name')
    def __init__(self, owner, module, config, tag_name, components):
        self.module_str = module
        self.tag_name = tag_name
        self.config = config

    @classmethod
    def reset(cls):
        FakeModuleBuilder.BUILD_STATE = "COMPLETE"
        FakeModuleBuilder.INSTANT_COMPLETE = False
        FakeModuleBuilder.on_build_cb = None
        FakeModuleBuilder.on_cancel_cb = None
        FakeModuleBuilder.on_finalize_cb = None
        FakeModuleBuilder.on_buildroot_add_artifacts_cb = None
        FakeModuleBuilder.on_tag_artifacts_cb = None
        FakeModuleBuilder.DEFAULT_GROUPS = None
        FakeModuleBuilder.backend = 'test'

    def buildroot_connect(self, groups):
        default_groups = FakeModuleBuilder.DEFAULT_GROUPS or {
            'srpm-build':
                set(['shadow-utils', 'fedora-release', 'redhat-rpm-config',
                     'rpm-build', 'fedpkg-minimal', 'gnupg2', 'bash']),
            'build':
                set(['unzip', 'fedora-release', 'tar', 'cpio', 'gawk',
                     'gcc', 'xz', 'sed', 'findutils', 'util-linux', 'bash',
                     'info', 'bzip2', 'grep', 'redhat-rpm-config',
                     'diffutils', 'make', 'patch', 'shadow-utils',
                     'coreutils', 'which', 'rpm-build', 'gzip', 'gcc-c++'])}
        if groups != default_groups:
            raise ValueError("Wrong groups in FakeModuleBuilder.buildroot_connect()")

    def buildroot_prep(self):
        pass

    def buildroot_resume(self):
        pass

    def buildroot_ready(self, artifacts=None):
        return True

    def buildroot_add_dependency(self, dependencies):
        pass

    def repo_from_tag(self, config, tag_name, arch):
        pass

    def buildroot_add_artifacts(self, artifacts, install=False):
        if FakeModuleBuilder.on_buildroot_add_artifacts_cb:
            FakeModuleBuilder.on_buildroot_add_artifacts_cb(self, artifacts, install)
        if self.backend == 'test':
            for nvr in artifacts:
                # buildroot_add_artifacts received a list of NVRs, but the tag message expects the
                # component name. At this point, the NVR may not be set if we are trying to reuse
                # all components, so we can't search the database. We must parse the package name
                # from the nvr and then tag it in the build tag. Kobo doesn't work when parsing
                # the NVR of a component with a module dist-tag, so we must manually do it.
                package_name = nvr.split('.module')[0].rsplit('-', 2)[0]
                # When INSTANT_COMPLETE is on, the components are already in the build tag
                if self.INSTANT_COMPLETE is False:
                    self._send_tag(package_name, nvr, dest_tag=False)
        elif self.backend == 'testlocal':
            self._send_repo_done()

    def buildroot_add_repos(self, dependencies):
        pass

    def tag_artifacts(self, artifacts, dest_tag=True):
        if FakeModuleBuilder.on_tag_artifacts_cb:
            FakeModuleBuilder.on_tag_artifacts_cb(self, artifacts, dest_tag=dest_tag)

        if self.backend == 'test':
            for nvr in artifacts:
                # tag_artifacts received a list of NVRs, but the tag message expects the
                # component name
                artifact = models.ComponentBuild.query.filter_by(nvr=nvr).first().package
                self._send_tag(artifact, nvr, dest_tag=dest_tag)

    @property
    def koji_session(self):
        session = Mock()

        def _newRepo(tag):
            session.newRepo = self._send_repo_done()
            return 123
        session.newRepo = _newRepo
        return session

    @property
    def module_build_tag(self):
        return {"name": self.tag_name + "-build"}

    def _send_repo_done(self):
        msg = module_build_service.messaging.KojiRepoChange(
            msg_id='a faked internal message',
            repo_tag=self.tag_name + "-build",
        )
        module_build_service.scheduler.consumer.work_queue_put(msg)

    def _send_tag(self, artifact, nvr, dest_tag=True):
        if dest_tag:
            tag = self.tag_name
        else:
            tag = self.tag_name + "-build"
        msg = module_build_service.messaging.KojiTagChange(
            msg_id='a faked internal message',
            tag=tag,
            artifact=artifact,
            nvr=nvr
        )
        module_build_service.scheduler.consumer.work_queue_put(msg)

    def _send_build_change(self, state, name, build_id):
        # build_id=1 and task_id=1 are OK here, because we are building just
        # one RPM at the time.
        msg = module_build_service.messaging.KojiBuildChange(
            msg_id='a faked internal message',
            build_id=build_id,
            task_id=build_id,
            build_name=name,
            build_new_state=state,
            build_release="1",
            build_version="1"
        )
        module_build_service.scheduler.consumer.work_queue_put(msg)

    def build(self, artifact_name, source):
        print("Starting building artifact %s: %s" % (artifact_name, source))
        build_id = randint(1000, 9999999)

        if FakeModuleBuilder.on_build_cb:
            FakeModuleBuilder.on_build_cb(self, artifact_name, source)

        if FakeModuleBuilder.BUILD_STATE != "BUILDING":
            self._send_build_change(
                koji.BUILD_STATES[FakeModuleBuilder.BUILD_STATE], artifact_name, build_id)

        reason = "Submitted %s to Koji" % (artifact_name)
        return build_id, koji.BUILD_STATES['BUILDING'], reason, None

    @staticmethod
    def get_disttag_srpm(disttag, module_build):
        # @FIXME
        return KojiModuleBuilder.get_disttag_srpm(disttag, module_build)

    def cancel_build(self, task_id):
        if FakeModuleBuilder.on_cancel_cb:
            FakeModuleBuilder.on_cancel_cb(self, task_id)

    def list_tasks_for_components(self, component_builds=None, state='active'):
        pass

    def recover_orphaned_artifact(self, component_build):
        msgs = []
        if self.INSTANT_COMPLETE:
            disttag = module_build_service.utils.get_rpm_release(
                component_build.module_build)
            # We don't know the version or release, so just use a random one here
            nvr = '{0}-1.0-1.{1}'.format(component_build.package, disttag)
            component_build.state = koji.BUILD_STATES['COMPLETE']
            component_build.nvr = nvr
            component_build.task_id = component_build.id + 51234
            component_build.state_reason = 'Found existing build'
            nvr_dict = kobo.rpmlib.parse_nvr(component_build.nvr)
            # Send a message stating the build is complete
            msgs.append(module_build_service.messaging.KojiBuildChange(
                'recover_orphaned_artifact: fake message', randint(1, 9999999),
                component_build.task_id, koji.BUILD_STATES['COMPLETE'], component_build.package,
                nvr_dict['version'], nvr_dict['release'], component_build.module_build.id))
            # Send a message stating that the build was tagged in the build tag
            msgs.append(module_build_service.messaging.KojiTagChange(
                'recover_orphaned_artifact: fake message',
                component_build.module_build.koji_tag + '-build', component_build.package,
                component_build.nvr))
        return msgs

    def finalize(self, succeeded=None):
        if FakeModuleBuilder.on_finalize_cb:
            FakeModuleBuilder.on_finalize_cb(self, succeeded)


def cleanup_moksha():
    # Necessary to restart the twisted reactor for the next test.
    import sys
    del sys.modules['twisted.internet.reactor']
    del sys.modules['moksha.hub.reactor']
    del sys.modules['moksha.hub']
    import moksha.hub.reactor # noqa


@patch('module_build_service.scheduler.handlers.modules.handle_stream_collision_modules')
@patch.object(module_build_service.config.Config, 'system', new_callable=PropertyMock,
              return_value='test')
@patch("module_build_service.builder.GenericBuilder.default_buildroot_groups",
       return_value={
           'srpm-build':
           set(['shadow-utils', 'fedora-release', 'redhat-rpm-config',
                'rpm-build', 'fedpkg-minimal', 'gnupg2', 'bash']),
           'build':
           set(['unzip', 'fedora-release', 'tar', 'cpio', 'gawk',
                'gcc', 'xz', 'sed', 'findutils', 'util-linux', 'bash',
                'info', 'bzip2', 'grep', 'redhat-rpm-config',
                'diffutils', 'make', 'patch', 'shadow-utils',
                'coreutils', 'which', 'rpm-build', 'gzip', 'gcc-c++'])})
class TestBuild:
    # Global variable used for tests if needed
    _global_var = None

    def setup_method(self, test_method):
        GenericBuilder.register_backend_class(FakeModuleBuilder)
        self.client = app.test_client()
        clean_database()

    def teardown_method(self, test_method):
        FakeModuleBuilder.reset()
        cleanup_moksha()
        for i in range(20):
            try:
                os.remove(build_logs.path(i))
            except Exception:
                pass

    @pytest.mark.parametrize('mmd_version', [1, 2])
    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    def test_submit_build(self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc,
                          mmd_version):
        """
        Tests the build of testmodule.yaml using FakeModuleBuilder which
        succeeds everytime.
        """
        if mmd_version == 1:
            yaml_file = 'testmodule.yaml'
        else:
            yaml_file = 'testmodule_v2.yaml'
        FakeSCM(mocked_scm, 'testmodule', yaml_file,
                '620ec77321b2ea7b0d67d82992dda3e1d67055b4')

        rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
            {'branch': 'master', 'scmurl': 'https://src.stg.fedoraproject.org/modules/'
                'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'}))

        data = json.loads(rv.data)
        module_build_id = data['id']

        # Check that components are tagged after the batch is built.
        tag_groups = []
        tag_groups.append(set(['perl-Tangerine-1-1', 'perl-List-Compare-1-1']))
        tag_groups.append(set(['tangerine-1-1']))

        def on_finalize_cb(cls, succeeded):
            assert succeeded is True

        def on_tag_artifacts_cb(cls, artifacts, dest_tag=True):
            assert tag_groups.pop(0) == set(artifacts)

        FakeModuleBuilder.on_finalize_cb = on_finalize_cb
        FakeModuleBuilder.on_tag_artifacts_cb = on_tag_artifacts_cb

        # Check that the components are added to buildroot after the batch
        # is built.
        buildroot_groups = []
        buildroot_groups.append(set(['module-build-macros-1-1']))
        buildroot_groups.append(set(['perl-Tangerine-1-1', 'perl-List-Compare-1-1']))
        buildroot_groups.append(set(['tangerine-1-1']))

        def on_buildroot_add_artifacts_cb(cls, artifacts, install):
            assert buildroot_groups.pop(0) == set(artifacts)

        FakeModuleBuilder.on_buildroot_add_artifacts_cb = on_buildroot_add_artifacts_cb

        msgs = []
        stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
        module_build_service.scheduler.main(msgs, stop)

        # All components should be built and module itself should be in "done"
        # or "ready" state.
        for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all():
            assert build.state == koji.BUILD_STATES['COMPLETE']
            assert build.module_build.state in [models.BUILD_STATES["done"],
                                                models.BUILD_STATES["ready"]]

        # All components has to be tagged, so tag_groups and buildroot_groups are empty...
        assert tag_groups == []
        assert buildroot_groups == []
        module_build = models.ModuleBuild.query.get(module_build_id)
        assert module_build.module_builds_trace[0].state == models.BUILD_STATES['init']
        assert module_build.module_builds_trace[1].state == models.BUILD_STATES['wait']
        assert module_build.module_builds_trace[2].state == models.BUILD_STATES['build']
        assert module_build.module_builds_trace[3].state == models.BUILD_STATES['done']
        assert module_build.module_builds_trace[4].state == models.BUILD_STATES['ready']
        assert len(module_build.module_builds_trace) == 5

    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    def test_submit_build_no_components(self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc):
        """
        Tests the build of a module with no components
        """
        FakeSCM(mocked_scm, 'python3', 'python3-no-components.yaml',
                '620ec77321b2ea7b0d67d82992dda3e1d67055b4')
        rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
            {'branch': 'master', 'scmurl': 'https://src.stg.fedoraproject.org/modules/'
                'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'}))

        data = json.loads(rv.data)
        module_build_id = data['id']
        msgs = []
        stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
        module_build_service.scheduler.main(msgs, stop)

        module_build = models.ModuleBuild.query.filter_by(id=module_build_id).one()
        # Make sure no component builds were registered
        assert len(module_build.component_builds) == 0
        # Make sure the build is done
        assert module_build.state == models.BUILD_STATES['ready']

    @patch('module_build_service.config.Config.check_for_eol',
           new_callable=PropertyMock, return_value=True)
    @patch('module_build_service.utils.submit._is_eol_in_pdc', return_value=True)
    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    def test_submit_build_eol_module(self, mocked_scm, mocked_get_user, is_eol, check,
                                     conf_system, dbg, hmsc):
        """ Tests the build of a module with an eol stream.  """
        FakeSCM(mocked_scm, 'python3', 'python3-no-components.yaml',
                '620ec77321b2ea7b0d67d82992dda3e1d67055b4')
        rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
            {'branch': 'master', 'scmurl': 'https://src.stg.fedoraproject.org/modules/'
                'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'}))

        assert rv.status_code == 400
        data = json.loads(rv.data)
        assert data['status'] == 400
        assert data['message'] == u'Module python3:master is marked as EOL in PDC.'

    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    def test_submit_build_from_yaml_not_allowed(
            self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc):
        FakeSCM(mocked_scm, "testmodule", "testmodule.yaml")

        testmodule = os.path.join(base_dir, 'staged_data', 'testmodule.yaml')
        with open(testmodule) as f:
            yaml = to_text_type(f.read())

        with patch.object(module_build_service.config.Config, 'yaml_submit_allowed',
                          new_callable=PropertyMock, return_value=False):
            rv = self.client.post('/module-build-service/1/module-builds/',
                                  content_type='multipart/form-data',
                                  data={'yaml': (testmodule, yaml)})
            data = json.loads(rv.data)
            assert data['status'] == 403
            assert data['message'] == 'YAML submission is not enabled'

    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    def test_submit_build_from_yaml_allowed(
            self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc):
        FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
                '620ec77321b2ea7b0d67d82992dda3e1d67055b4')
        testmodule = os.path.join(base_dir, 'staged_data', 'testmodule.yaml')

        with patch.object(module_build_service.config.Config, 'yaml_submit_allowed',
                          new_callable=PropertyMock, return_value=True):
            with open(testmodule, 'rb') as f:
                yaml_file = FileStorage(f)
                rv = self.client.post('/module-build-service/1/module-builds/',
                                      content_type='multipart/form-data',
                                      data={'yaml': yaml_file})
            data = json.loads(rv.data)
            assert data['id'] == 2
        msgs = []
        stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
        module_build_service.scheduler.main(msgs, stop)
        assert models.ModuleBuild.query.first().state == models.BUILD_STATES['ready']

    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    def test_submit_build_cancel(self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc):
        """
        Submit all builds for a module and cancel the module build later.
        """
        FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
                '620ec77321b2ea7b0d67d82992dda3e1d67055b4')

        rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
            {'branch': 'master', 'scmurl': 'https://src.stg.fedoraproject.org/modules/'
                'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'}))

        data = json.loads(rv.data)
        module_build_id = data['id']

        # This callback is called before return of FakeModuleBuilder.build()
        # method. We just cancel the build here using the web API to simulate
        # user cancelling the build in the middle of building.
        def on_build_cb(cls, artifact_name, source):
            self.client.patch('/module-build-service/1/module-builds/' + str(module_build_id),
                              data=json.dumps({'state': 'failed'}))

        cancelled_tasks = []

        def on_cancel_cb(cls, task_id):
            cancelled_tasks.append(task_id)

        def on_finalize_cb(cls, succeeded):
            assert succeeded is False

        # We do not want the builds to COMPLETE, but instead we want them
        # to be in the BULDING state after the FakeModuleBuilder.build().
        FakeModuleBuilder.BUILD_STATE = "BUILDING"
        FakeModuleBuilder.on_build_cb = on_build_cb
        FakeModuleBuilder.on_cancel_cb = on_cancel_cb
        FakeModuleBuilder.on_finalize_cb = on_finalize_cb

        msgs = []
        stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
        module_build_service.scheduler.main(msgs, stop)

        # Because we did not finished single component build and canceled the
        # module build, all components and even the module itself should be in
        # failed state with state_reason se to cancellation message.
        for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all():
            assert build.state == koji.BUILD_STATES['FAILED']
            assert build.state_reason == "Canceled by Homer J. Simpson."
            assert build.module_build.state == models.BUILD_STATES["failed"]
            assert build.module_build.state_reason == "Canceled by Homer J. Simpson."

            # Check that cancel_build has been called for this build
            if build.task_id:
                assert build.task_id in cancelled_tasks

    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    def test_submit_build_instant_complete(
            self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc):
        """
        Tests the build of testmodule.yaml using FakeModuleBuilder which
        succeeds everytime.
        """
        FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
                '620ec77321b2ea7b0d67d82992dda3e1d67055b4')

        rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
            {'branch': 'master', 'scmurl': 'https://src.stg.fedoraproject.org/modules/'
                'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'}))

        data = json.loads(rv.data)
        module_build_id = data['id']
        FakeModuleBuilder.INSTANT_COMPLETE = True

        msgs = []
        stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
        module_build_service.scheduler.main(msgs, stop)

        # All components should be built and module itself should be in "done"
        # or "ready" state.
        for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all():
            assert build.state == koji.BUILD_STATES['COMPLETE']
            assert build.module_build.state in [models.BUILD_STATES["done"],
                                                models.BUILD_STATES["ready"]]

    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    @patch("module_build_service.config.Config.num_concurrent_builds",
           new_callable=PropertyMock, return_value=1)
    def test_submit_build_concurrent_threshold(self, conf_num_concurrent_builds,
                                               mocked_scm, mocked_get_user,
                                               conf_system, dbg, hmsc):
        """
        Tests the build of testmodule.yaml using FakeModuleBuilder with
        num_concurrent_builds set to 1.
        """
        FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
                '620ec77321b2ea7b0d67d82992dda3e1d67055b4')

        rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
            {'branch': 'master', 'scmurl': 'https://src.stg.fedoraproject.org/modules/'
                'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'}))

        data = json.loads(rv.data)
        module_build_id = data['id']

        def stop(message):
            """
            Stop the scheduler when the module is built or when we try to build
            more components than the num_concurrent_builds.
            """
            main_stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
            over_threshold = conf.num_concurrent_builds < \
                db.session.query(models.ComponentBuild).filter_by(
                    state=koji.BUILD_STATES['BUILDING']).count()
            return main_stop(message) or over_threshold

        msgs = []
        module_build_service.scheduler.main(msgs, stop)

        # All components should be built and module itself should be in "done"
        # or "ready" state.
        for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all():
            assert build.state == koji.BUILD_STATES['COMPLETE']
            # When this fails, it can mean that num_concurrent_builds
            # threshold has been met.
            assert build.module_build.state in [models.BUILD_STATES["done"],
                                                models.BUILD_STATES["ready"]]

    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    @patch("module_build_service.config.Config.num_concurrent_builds",
           new_callable=PropertyMock, return_value=2)
    def test_try_to_reach_concurrent_threshold(self, conf_num_concurrent_builds,
                                               mocked_scm, mocked_get_user,
                                               conf_system, dbg, hmsc):
        """
        Tests that we try to submit new component build right after
        the previous one finished without waiting for all
        the num_concurrent_builds to finish.
        """
        FakeSCM(mocked_scm, 'testmodule-more-components', 'testmodule-more-components.yaml',
                '620ec77321b2ea7b0d67d82992dda3e1d67055b4')
        self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
            {'branch': 'master', 'scmurl': 'https://src.stg.fedoraproject.org/modules/'
                'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'}))

        # Holds the number of concurrent component builds during
        # the module build.
        TestBuild._global_var = []

        def stop(message):
            """
            Stop the scheduler when the module is built or when we try to build
            more components than the num_concurrent_builds.
            """
            main_stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
            num_building = db.session.query(models.ComponentBuild).filter_by(
                state=koji.BUILD_STATES['BUILDING']).count()
            over_threshold = conf.num_concurrent_builds < num_building
            TestBuild._global_var.append(num_building)
            return main_stop(message) or over_threshold

        msgs = []
        module_build_service.scheduler.main(msgs, stop)

        # _global_var looks similar to this: [0, 1, 0, 0, 2, 2, 1, 0, 0, 0]
        # It shows the number of concurrent builds in the time. At first we
        # want to remove adjacent duplicate entries, because we only care
        # about changes.
        # We are building two batches, so there should be just two situations
        # when we should be building just single component:
        #   1) module-base-macros in first batch.
        #   2) The last component of second batch.
        # If we are building single component more often, num_concurrent_builds
        # does not work correctly.
        num_builds = [k for k, g in itertools.groupby(TestBuild._global_var)]
        assert num_builds.count(1) == 2

    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    @patch("module_build_service.config.Config.num_concurrent_builds",
           new_callable=PropertyMock, return_value=1)
    def test_build_in_batch_fails(self, conf_num_concurrent_builds, mocked_scm,
                                  mocked_get_user, conf_system, dbg, hmsc):
        """
        Tests that if the build in batch fails, other components in a batch
        are still build, but next batch is not started.
        """
        FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
                '620ec77321b2ea7b0d67d82992dda3e1d67055b4')

        rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
            {'branch': 'master', 'scmurl': 'https://src.stg.fedoraproject.org/modules/'
                'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'}))

        data = json.loads(rv.data)
        module_build_id = data['id']

        def on_build_cb(cls, artifact_name, source):
            # fail perl-Tangerine build
            if artifact_name.startswith("perl-Tangerine"):
                FakeModuleBuilder.BUILD_STATE = "FAILED"
            else:
                FakeModuleBuilder.BUILD_STATE = "COMPLETE"

        FakeModuleBuilder.on_build_cb = on_build_cb

        # Check that no components are tagged when single component fails
        # in batch.
        def on_tag_artifacts_cb(cls, artifacts, dest_tag=True):
            raise ValueError("No component should be tagged.")
        FakeModuleBuilder.on_tag_artifacts_cb = on_tag_artifacts_cb

        msgs = []
        stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
        module_build_service.scheduler.main(msgs, stop)

        for c in models.ComponentBuild.query.filter_by(module_id=module_build_id).all():
            # perl-Tangerine is expected to fail as configured in on_build_cb.
            if c.package == "perl-Tangerine":
                assert c.state == koji.BUILD_STATES['FAILED']
            # tangerine is expected to fail, because it is in batch 3, but
            # we had a failing component in batch 2.
            elif c.package == "tangerine":
                assert c.state == koji.BUILD_STATES['FAILED']
                assert c.state_reason == "Component(s) perl-Tangerine failed to build."
            else:
                assert c.state == koji.BUILD_STATES['COMPLETE']

            # Whole module should be failed.
            assert c.module_build.state == models.BUILD_STATES['failed']
            assert c.module_build.state_reason == "Component(s) perl-Tangerine failed to build."

            # We should end up with batch 2 and never start batch 3, because
            # there were failed components in batch 2.
            assert c.module_build.batch == 2

    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    @patch("module_build_service.config.Config.num_concurrent_builds",
           new_callable=PropertyMock, return_value=1)
    def test_all_builds_in_batch_fail(self, conf_num_concurrent_builds, mocked_scm,
                                      mocked_get_user, conf_system, dbg, hmsc):
        """
        Tests that if the build in batch fails, other components in a batch
        are still build, but next batch is not started.
        """
        FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
                '620ec77321b2ea7b0d67d82992dda3e1d67055b4')

        rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
            {'branch': 'master', 'scmurl': 'https://src.stg.fedoraproject.org/modules/'
                'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'}))

        data = json.loads(rv.data)
        module_build_id = data['id']

        def on_build_cb(cls, artifact_name, source):
            # Next components *after* the module-build-macros will fail
            # to build.
            if not artifact_name.startswith("module-build-macros"):
                FakeModuleBuilder.BUILD_STATE = "FAILED"

        FakeModuleBuilder.on_build_cb = on_build_cb

        msgs = []
        stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
        module_build_service.scheduler.main(msgs, stop)

        for c in models.ComponentBuild.query.filter_by(module_id=module_build_id).all():
            # perl-Tangerine is expected to fail as configured in on_build_cb.
            if c.package == "module-build-macros":
                assert c.state == koji.BUILD_STATES['COMPLETE']
            else:
                assert c.state == koji.BUILD_STATES['FAILED']

            # Whole module should be failed.
            assert c.module_build.state == models.BUILD_STATES['failed']
            assert re.match(r'Component\(s\) (perl-Tangerine|perl-List-Compare), '
                            '(perl-Tangerine|perl-List-Compare) failed to build.',
                            c.module_build.state_reason)

            # We should end up with batch 2 and never start batch 3, because
            # there were failed components in batch 2.
            assert c.module_build.batch == 2

    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    def test_submit_build_reuse_all(self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc):
        """
        Tests that we do not try building module-build-macros when reusing all
        components in a module build.
        """
        reuse_component_init_data()

        def on_build_cb(cls, artifact_name, source):
            raise ValueError("All components should be reused, not build.")
        FakeModuleBuilder.on_build_cb = on_build_cb

        # Check that components are tagged after the batch is built.
        tag_groups = []
        tag_groups.append(set(
            ['perl-Tangerine-0.23-1.module+0+d027b723',
             'perl-List-Compare-0.53-5.module+0+d027b723',
             'tangerine-0.22-3.module+0+d027b723']))

        def on_tag_artifacts_cb(cls, artifacts, dest_tag=True):
            if dest_tag is True:
                assert tag_groups.pop(0) == set(artifacts)
        FakeModuleBuilder.on_tag_artifacts_cb = on_tag_artifacts_cb

        buildtag_groups = []
        buildtag_groups.append(set(
            ['perl-Tangerine-0.23-1.module+0+d027b723',
             'perl-List-Compare-0.53-5.module+0+d027b723',
             'tangerine-0.22-3.module+0+d027b723']))

        def on_buildroot_add_artifacts_cb(cls, artifacts, install):
            assert buildtag_groups.pop(0) == set(artifacts)
        FakeModuleBuilder.on_buildroot_add_artifacts_cb = on_buildroot_add_artifacts_cb

        msgs = [MBSModule("local module build", 3, 1)]
        stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
        module_build_service.scheduler.main(msgs, stop)

        reused_component_ids = {"module-build-macros": None, "tangerine": 3,
                                "perl-Tangerine": 1, "perl-List-Compare": 2}

        # All components should be built and module itself should be in "done"
        # or "ready" state.
        for build in models.ComponentBuild.query.filter_by(module_id=3).all():
            assert build.state == koji.BUILD_STATES['COMPLETE']
            assert build.module_build.state in [models.BUILD_STATES["done"],
                                                models.BUILD_STATES["ready"]]
            assert build.reused_component_id == reused_component_ids[build.package]

    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    def test_submit_build_reuse_all_without_build_macros(self, mocked_scm, mocked_get_user,
                                                         conf_system, dbg, hmsc):
        """
        Tests that we can reuse components even when the reused module does
        not have module-build-macros component.
        """
        reuse_component_init_data()

        models.ComponentBuild.query.filter_by(package="module-build-macros").delete()
        assert len(models.ComponentBuild.query.filter_by(package="module-build-macros").all()) == 0

        db.session.commit()

        def on_build_cb(cls, artifact_name, source):
            raise ValueError("All components should be reused, not build.")
        FakeModuleBuilder.on_build_cb = on_build_cb

        # Check that components are tagged after the batch is built.
        tag_groups = []
        tag_groups.append(set(
            ['perl-Tangerine-0.23-1.module+0+d027b723',
             'perl-List-Compare-0.53-5.module+0+d027b723',
             'tangerine-0.22-3.module+0+d027b723']))

        def on_tag_artifacts_cb(cls, artifacts, dest_tag=True):
            if dest_tag is True:
                assert tag_groups.pop(0) == set(artifacts)
        FakeModuleBuilder.on_tag_artifacts_cb = on_tag_artifacts_cb

        buildtag_groups = []
        buildtag_groups.append(set(
            ['perl-Tangerine-0.23-1.module+0+d027b723',
             'perl-List-Compare-0.53-5.module+0+d027b723',
             'tangerine-0.22-3.module+0+d027b723']))

        def on_buildroot_add_artifacts_cb(cls, artifacts, install):
            assert buildtag_groups.pop(0) == set(artifacts)
        FakeModuleBuilder.on_buildroot_add_artifacts_cb = on_buildroot_add_artifacts_cb

        msgs = [MBSModule("local module build", 3, 1)]
        stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
        module_build_service.scheduler.main(msgs, stop)

        # All components should be built and module itself should be in "done"
        # or "ready" state.
        for build in models.ComponentBuild.query.filter_by(module_id=3).all():
            assert build.state == koji.BUILD_STATES['COMPLETE']
            assert build.module_build.state in [models.BUILD_STATES["done"],
                                                models.BUILD_STATES["ready"]]
            assert build.package != "module-build-macros"

    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    def test_submit_build_resume(self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc):
        """
        Tests that resuming the build works even when previous batches
        are already built.
        """
        now = datetime.utcnow()
        submitted_time = now - timedelta(minutes=3)
        # Create a module in the failed state
        build_one = models.ModuleBuild()
        build_one.name = 'testmodule'
        build_one.stream = 'master'
        build_one.version = '2820180205135154'
        build_one.build_context = 'return_runtime_context'
        build_one.ref_build_context = 'return_runtime_context'
        build_one.runtime_context = '9c690d0e'
        build_one.context = '9c690d0e'
        build_one.state = models.BUILD_STATES['failed']
        current_dir = os.path.dirname(__file__)
        formatted_testmodule_yml_path = os.path.join(
            current_dir, '..', 'staged_data', 'formatted_testmodule.yaml')
        with open(formatted_testmodule_yml_path, 'r') as f:
            build_one.modulemd = to_text_type(f.read())
        build_one.koji_tag = 'module-testmodule-master-20180205135154-9c690d0e'
        build_one.scmurl = 'https://src.stg.fedoraproject.org/modules/testmodule.git?#7fea453'
        build_one.batch = 2
        build_one.owner = 'Homer J. Simpson'
        build_one.time_submitted = submitted_time
        build_one.time_modified = now
        build_one.rebuild_strategy = 'changed-and-after'
        # It went from init, to wait, to build, and then failed
        mbt_one = models.ModuleBuildTrace(
            state_time=submitted_time, state=models.BUILD_STATES['init'])
        mbt_two = models.ModuleBuildTrace(
            state_time=now - timedelta(minutes=2), state=models.BUILD_STATES['wait'])
        mbt_three = models.ModuleBuildTrace(
            state_time=now - timedelta(minutes=1), state=models.BUILD_STATES['build'])
        mbt_four = models.ModuleBuildTrace(state_time=now, state=build_one.state)
        build_one.module_builds_trace.append(mbt_one)
        build_one.module_builds_trace.append(mbt_two)
        build_one.module_builds_trace.append(mbt_three)
        build_one.module_builds_trace.append(mbt_four)
        # Successful component
        component_one = models.ComponentBuild()
        component_one.package = 'perl-Tangerine'
        component_one.format = 'rpms'
        component_one.scmurl = 'https://src.stg.fedoraproject.org/rpms/perl-Tangerine.git?#master'
        component_one.state = koji.BUILD_STATES['COMPLETE']
        component_one.nvr = 'perl-Tangerine-0:0.22-2.module+0+d027b723'
        component_one.batch = 2
        component_one.module_id = 2
        component_one.ref = '7e96446223f1ad84a26c7cf23d6591cd9f6326c6'
        component_one.tagged = True
        component_one.tagged_in_final = True
        # Failed component
        component_two = models.ComponentBuild()
        component_two.package = 'perl-List-Compare'
        component_two.format = 'rpms'
        component_two.scmurl = \
            'https://src.stg.fedoraproject.org/rpms/perl-List-Compare.git?#master'
        component_two.state = koji.BUILD_STATES['FAILED']
        component_two.batch = 2
        component_two.module_id = 2
        # Component that isn't started yet
        component_three = models.ComponentBuild()
        component_three.package = 'tangerine'
        component_three.format = 'rpms'
        component_three.scmurl = 'https://src.stg.fedoraproject.org/rpms/tangerine.git?#master'
        component_three.batch = 3
        component_three.module_id = 2
        # module-build-macros
        component_four = models.ComponentBuild()
        component_four.package = 'module-build-macros'
        component_four.format = 'rpms'
        component_four.state = koji.BUILD_STATES['COMPLETE']
        component_four.scmurl = (
            '/tmp/module_build_service-build-macrosqr4AWH/SRPMS/module-build-macros-0.1-1.'
            'module_testmodule_master_20170109091357.src.rpm')
        component_four.batch = 1
        component_four.module_id = 2
        component_four.tagged = True
        component_four.build_time_only = True

        db.session.add(build_one)
        db.session.add(component_one)
        db.session.add(component_two)
        db.session.add(component_three)
        db.session.add(component_four)
        db.session.commit()
        db.session.expire_all()

        FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
                '620ec77321b2ea7b0d67d82992dda3e1d67055b4')
        # Resubmit the failed module
        rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
            {'branch': 'master', 'scmurl': 'https://src.stg.fedoraproject.org/modules/'
                'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'}))

        data = json.loads(rv.data)
        module_build_id = data['id']
        module_build = models.ModuleBuild.query.filter_by(id=module_build_id).one()
        components = models.ComponentBuild.query.filter_by(
            module_id=module_build_id, batch=2).order_by(models.ComponentBuild.id).all()
        # Make sure the build went from failed to wait
        assert module_build.state == models.BUILD_STATES['wait']
        assert module_build.state_reason == 'Resubmitted by Homer J. Simpson'
        # Make sure the state was reset on the failed component
        assert components[1].state is None
        db.session.expire_all()

        # Run the backend
        msgs = []
        stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
        module_build_service.scheduler.main(msgs, stop)

        # All components should be built and module itself should be in "done"
        # or "ready" state.
        for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all():
            assert build.state == koji.BUILD_STATES['COMPLETE']
            assert build.module_build.state in [models.BUILD_STATES['done'],
                                                models.BUILD_STATES['ready']]

    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    def test_submit_build_resume_recover_orphaned_macros(
            self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc):
        """
        Tests that resuming the build works when module-build-macros is orphaned but marked as
        failed in the database
        """
        FakeModuleBuilder.INSTANT_COMPLETE = True
        now = datetime.utcnow()
        submitted_time = now - timedelta(minutes=3)
        # Create a module in the failed state
        build_one = models.ModuleBuild()
        build_one.name = 'testmodule'
        build_one.stream = 'master'
        build_one.version = '2820180205135154'
        build_one.build_context = 'return_runtime_context'
        build_one.ref_build_context = 'return_runtime_context'
        build_one.runtime_context = '9c690d0e'
        build_one.state = models.BUILD_STATES['failed']
        # this is not calculated by real but just a value to
        # match the calculated context from expanded test mmd
        build_one.context = '9c690d0e'
        current_dir = os.path.dirname(__file__)
        formatted_testmodule_yml_path = os.path.join(
            current_dir, '..', 'staged_data', 'formatted_testmodule.yaml')
        with open(formatted_testmodule_yml_path, 'r') as f:
            build_one.modulemd = to_text_type(f.read())
        build_one.koji_tag = 'module-testmodule-master-20180205135154-6ef9a711'
        build_one.scmurl = 'https://src.stg.fedoraproject.org/modules/testmodule.git?#7fea453'
        build_one.batch = 2
        build_one.owner = 'Homer J. Simpson'
        build_one.time_submitted = submitted_time
        build_one.time_modified = now
        build_one.rebuild_strategy = 'changed-and-after'
        # It went from init, to wait, to build, and then failed
        mbt_one = models.ModuleBuildTrace(
            state_time=submitted_time, state=models.BUILD_STATES['init'])
        mbt_two = models.ModuleBuildTrace(
            state_time=now - timedelta(minutes=2), state=models.BUILD_STATES['wait'])
        mbt_three = models.ModuleBuildTrace(
            state_time=now - timedelta(minutes=1), state=models.BUILD_STATES['build'])
        mbt_four = models.ModuleBuildTrace(state_time=now, state=build_one.state)
        build_one.module_builds_trace.append(mbt_one)
        build_one.module_builds_trace.append(mbt_two)
        build_one.module_builds_trace.append(mbt_three)
        build_one.module_builds_trace.append(mbt_four)
        # Components that haven't started yet
        component_one = models.ComponentBuild()
        component_one.package = 'perl-Tangerine'
        component_one.format = 'rpms'
        component_one.scmurl = 'https://src.stg.fedoraproject.org/rpms/perl-Tangerine.git?#master'
        component_one.batch = 2
        component_one.module_id = 2
        component_two = models.ComponentBuild()
        component_two.package = 'perl-List-Compare'
        component_two.format = 'rpms'
        component_two.scmurl = \
            'https://src.stg.fedoraproject.org/rpms/perl-List-Compare.git?#master'
        component_two.batch = 2
        component_two.module_id = 2
        component_three = models.ComponentBuild()
        component_three.package = 'tangerine'
        component_three.format = 'rpms'
        component_three.scmurl = 'https://src.stg.fedoraproject.org/rpms/tangerine.git?#master'
        component_three.batch = 3
        component_three.module_id = 2
        # Failed module-build-macros
        component_four = models.ComponentBuild()
        component_four.package = 'module-build-macros'
        component_four.format = 'rpms'
        component_four.state = koji.BUILD_STATES['FAILED']
        component_four.scmurl = (
            '/tmp/module_build_service-build-macrosqr4AWH/SRPMS/module-build-macros-0.1-1.'
            'module_testmodule_master_20180205135154.src.rpm')
        component_four.batch = 1
        component_four.module_id = 2
        component_four.build_time_only = True

        db.session.add(build_one)
        db.session.add(component_one)
        db.session.add(component_two)
        db.session.add(component_three)
        db.session.add(component_four)
        db.session.commit()
        db.session.expire_all()

        FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml', '7fea453')
        # Resubmit the failed module
        rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
            {'branch': 'master', 'scmurl': 'https://src.stg.fedoraproject.org/modules/'
                'testmodule.git?#7fea453'}))

        data = json.loads(rv.data)
        module_build_id = data['id']
        module_build = models.ModuleBuild.query.filter_by(id=module_build_id).one()
        # Make sure the build went from failed to wait
        assert module_build.state == models.BUILD_STATES['wait']
        assert module_build.state_reason == 'Resubmitted by Homer J. Simpson'
        # Make sure the state was reset on the failed component
        for c in module_build.component_builds:
            assert c.state is None
        db.session.expire_all()

        # Run the backend
        msgs = []
        stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
        module_build_service.scheduler.main(msgs, stop)

        # All components should be built and module itself should be in "done"
        # or "ready" state.
        for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all():
            assert build.state == koji.BUILD_STATES['COMPLETE']
            assert build.module_build.state in [models.BUILD_STATES['done'],
                                                models.BUILD_STATES['ready']]

    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    def test_submit_build_resume_failed_init(
            self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc):
        """
        Tests that resuming the build works when the build failed during the init step
        """
        FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
                '620ec77321b2ea7b0d67d82992dda3e1d67055b4')
        stop = module_build_service.scheduler.make_simple_stop_condition(db.session)

        with patch('module_build_service.utils.submit.format_mmd') as mock_format_mmd:
            mock_format_mmd.side_effect = Forbidden(
                'Custom component repositories aren\'t allowed.')
            rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
                {'branch': 'master', 'scmurl': 'https://src.stg.fedoraproject.org/modules/'
                    'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'}))
            # Run the backend so that it fails in the "init" handler
            module_build_service.scheduler.main([], stop)
            cleanup_moksha()

        module_build_id = json.loads(rv.data)['id']
        module_build = models.ModuleBuild.query.filter_by(id=module_build_id).one()
        assert module_build.state == models.BUILD_STATES['failed']
        assert module_build.state_reason == 'Custom component repositories aren\'t allowed.'
        assert len(module_build.module_builds_trace) == 2
        assert module_build.module_builds_trace[0].state == models.BUILD_STATES['init']
        assert module_build.module_builds_trace[1].state == models.BUILD_STATES['failed']

        # Resubmit the failed module
        rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
            {'branch': 'master',
             'scmurl': ('https://src.stg.fedoraproject.org/modules/testmodule.git?'
                        '#620ec77321b2ea7b0d67d82992dda3e1d67055b4')}))

        module_build = models.ModuleBuild.query.filter_by(id=module_build_id).one()
        components = models.ComponentBuild.query.filter_by(
            module_id=module_build_id, batch=2).order_by(models.ComponentBuild.id).all()
        # Make sure the build went from failed to init
        assert module_build.state == models.BUILD_STATES['init']
        assert module_build.state_reason == 'Resubmitted by Homer J. Simpson'
        # Make sure there are no components
        assert components == []
        db.session.expire_all()

        # Run the backend again
        module_build_service.scheduler.main([], stop)

        # All components should be built and module itself should be in "done"
        # or "ready" state.
        for build in models.ComponentBuild.query.filter_by(module_id=module_build_id).all():
            assert build.state == koji.BUILD_STATES['COMPLETE']
            assert build.module_build.state in [models.BUILD_STATES['done'],
                                                models.BUILD_STATES['ready']]

    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    def test_submit_build_resume_init_fail(
            self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc):
        """
        Tests that resuming the build fails when the build is in init state
        """
        FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
                '620ec77321b2ea7b0d67d82992dda3e1d67055b4')
        # Post so a module is in the init phase
        rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
            {'branch': 'master', 'scmurl': 'https://src.stg.fedoraproject.org/modules/'
                'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'}))
        assert rv.status_code == 201
        # Run the backend
        stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
        module_build_service.scheduler.main([], stop)
        # Post again and make sure it fails
        rv2 = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
            {'branch': 'master', 'scmurl': 'https://src.stg.fedoraproject.org/modules/'
                'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'}))
        data = json.loads(rv2.data)
        expected = {
            'error': 'Conflict',
            'message': ('Module (state=5) already exists. Only a new build, resubmission of a '
                        'failed build or build against new buildrequirements is allowed.'),
            'status': 409
        }
        assert data == expected

    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    def test_submit_build_repo_regen_not_started_batch(self, mocked_scm, mocked_get_user,
                                                       conf_system, dbg, hmsc):
        """
        Tests that if MBS starts a new batch, the concurrent component threshold is met before a
        build can start, and an unexpected repo regen occurs, the build will not fail.

        See: https://pagure.io/fm-orchestrator/issue/864
        """
        FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
                '620ec77321b2ea7b0d67d82992dda3e1d67055b4')

        rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(
            {'branch': 'master', 'scmurl': 'https://src.stg.fedoraproject.org/modules/'
                'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'}))

        data = json.loads(rv.data)
        module_build_id = data['id']

        def _stop_condition(message):
            # Stop the backend if the module batch is 2 (where we simulate the concurrent threshold
            # being met). For safety, also stop the backend if the module erroneously completes.
            module = db.session.query(models.ModuleBuild).get(module_build_id)
            return module.batch == 2 or module.state >= models.BUILD_STATES['done']

        with patch('module_build_service.utils.batches.at_concurrent_component_threshold') as \
                mock_acct:
            # Once we get to batch 2, then simulate the concurrent threshold being met
            def _at_concurrent_component_threshold(config, session):
                return db.session.query(models.ModuleBuild).get(module_build_id).batch == 2
            mock_acct.side_effect = _at_concurrent_component_threshold
            module_build_service.scheduler.main([], _stop_condition)

        # Only module-build-macros should be built
        for build in db.session.query(models.ComponentBuild).filter_by(
                module_id=module_build_id).all():
            if build.package == 'module-build-macros':
                assert build.state == koji.BUILD_STATES['COMPLETE']
            else:
                assert build.state is None
            assert build.module_build.state == models.BUILD_STATES['build']

        # Simulate a random repo regen message that MBS didn't expect
        cleanup_moksha()
        module = db.session.query(models.ModuleBuild).get(module_build_id)
        msgs = [module_build_service.messaging.KojiRepoChange(
            msg_id='a faked internal message', repo_tag=module.koji_tag + '-build')]
        db.session.expire_all()
        # Stop after processing the seeded message
        module_build_service.scheduler.main(msgs, lambda message: True)
        # Make sure the module build didn't fail so that the poller can resume it later
        module = db.session.query(models.ModuleBuild).get(module_build_id)
        assert module.state == models.BUILD_STATES['build']


@patch("module_build_service.config.Config.system",
       new_callable=PropertyMock, return_value="testlocal")
class TestLocalBuild:

    def setup_method(self, test_method):
        FakeModuleBuilder.on_build_cb = None
        FakeModuleBuilder.backend = 'testlocal'
        GenericBuilder.register_backend_class(FakeModuleBuilder)
        self.client = app.test_client()
        clean_database()

    def teardown_method(self, test_method):
        FakeModuleBuilder.reset()
        cleanup_moksha()
        for i in range(20):
            try:
                os.remove(build_logs.path(i))
            except Exception:
                pass

    @patch('module_build_service.scheduler.handlers.modules.handle_stream_collision_modules')
    @patch('module_build_service.auth.get_user', return_value=user)
    @patch('module_build_service.scm.SCM')
    @patch("module_build_service.config.Config.mock_resultsdir",
           new_callable=PropertyMock,
           return_value=path.join(base_dir, 'staged_data', "local_builds"))
    def test_submit_build_local_dependency(
            self, resultsdir, mocked_scm, mocked_get_user, conf_system, hmsc):
        """
        Tests local module build dependency.
        """
        with app.app_context():
            module_build_service.utils.load_local_builds(["platform"])
            FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',
                    '620ec77321b2ea7b0d67d82992dda3e1d67055b4')

            rv = self.client.post(
                '/module-build-service/1/module-builds/', data=json.dumps(
                    {'branch': 'master',
                     'scmurl': 'https://src.stg.fedoraproject.org/modules/'
                     'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'}))

            data = json.loads(rv.data)
            module_build_id = data['id']

            # Local base-runtime has changed profiles, so we can detect we use
            # the local one and not the main one.
            FakeModuleBuilder.DEFAULT_GROUPS = {
                'srpm-build':
                    set(['bar']),
                'build':
                    set(['foo'])}

            msgs = []
            stop = module_build_service.scheduler.make_simple_stop_condition(
                db.session)
            module_build_service.scheduler.main(msgs, stop)

            # All components should be built and module itself should be in "done"
            # or "ready" state.
            for build in models.ComponentBuild.query.filter_by(
                    module_id=module_build_id).all():
                assert build.state == koji.BUILD_STATES['COMPLETE']
                assert build.module_build.state in [models.BUILD_STATES["done"],
                                                    models.BUILD_STATES["ready"]]