Do not build module-build-macros when all the components will be reused from previous module build.

module_build_service/models.py

@@ -133,6 +133,26 @@ class ModuleBuild(MBSBase):
                 if component.batch == self.batch
             ]
 
+    def up_to_current_batch(self, state=None):
+        """
+        Returns all components of this module in the current batch and
+        in the previous batches.
+        """
+
+        if not self.batch:
+            raise ValueError("No batch is in progress: %r" % self.batch)
+
+        if state != None:
+            return [
+                component for component in self.component_builds
+                if component.batch <= self.batch and component.state == state
+            ]
+        else:
+            return [
+                component for component in self.component_builds
+                if component.batch <= self.batch
+            ]
+
     def mmd(self):
         mmd = _modulemd.ModuleMetadata()
         try:

module_build_service/scheduler/handlers/modules.py

@@ -28,6 +28,8 @@ import module_build_service.builder
 import module_build_service.pdc
 import module_build_service.utils
 import module_build_service.messaging
+from module_build_service.utils import (
+    start_next_batch_build, attempt_to_reuse_all_components)
 
 from requests.exceptions import ConnectionError
 
@@ -234,32 +236,42 @@ def wait(config, session, msg):
     log.debug("Adding dependencies %s into buildroot for module %s" % (dependencies, module_info))
     builder.buildroot_add_repos(dependencies)
 
-    # inject dist-tag into buildroot
-    srpm = builder.get_disttag_srpm(
-        disttag=".%s" % get_rpm_release_from_mmd(build.mmd()),
-        module_build=build)
+    # If all components in module build will be reused, we don't have to build
+    # module-build-macros, because there won't be any build done.
+    if attempt_to_reuse_all_components(builder, session, build):
+        log.info("All components have ben reused for module %r, "
+                 "skipping build" % build)
+        session.commit()
+        return []
+    else:
+        # Build the module-build-macros
+        # inject dist-tag into buildroot
+        srpm = builder.get_disttag_srpm(
+            disttag=".%s" % get_rpm_release_from_mmd(build.mmd()),
+            module_build=build)
 
-    log.debug("Starting build batch 1")
-    build.batch = 1
+        log.debug("Starting build batch 1")
+        build.batch = 1
+        session.commit()
 
-    artifact_name = "module-build-macros"
-    task_id, state, reason, nvr = builder.build(artifact_name=artifact_name, source=srpm)
+        artifact_name = "module-build-macros"
+        task_id, state, reason, nvr = builder.build(artifact_name=artifact_name, source=srpm)
 
-    component_build = models.ComponentBuild(
-        module_id=build.id,
-        package=artifact_name,
-        format="rpms",
-        scmurl=srpm,
-        task_id=task_id,
-        state=state,
-        state_reason=reason,
-        nvr=nvr,
-        batch=1,
-    )
-    session.add(component_build)
-    build.transition(config, state="build")
-    session.add(build)
-    session.commit()
+        component_build = models.ComponentBuild(
+            module_id=build.id,
+            package=artifact_name,
+            format="rpms",
+            scmurl=srpm,
+            task_id=task_id,
+            state=state,
+            state_reason=reason,
+            nvr=nvr,
+            batch=1,
+        )
+        session.add(component_build)
+        build.transition(config, state="build")
+        session.add(build)
+        session.commit()
 
     # If this build already exists and is done, then fake the repo change event
     # back to the scheduler

module_build_service/scheduler/handlers/tags.py

@@ -72,10 +72,10 @@ def tagged(config, session, msg):
                  "building components in a batch", tag)
         return []
 
-    # Get the list of untagged components in current batch which
+    # Get the list of untagged components in current/previous batches which
     # have been built successfully.
     untagged_components = [
-        c for c in module_build.current_batch()
+        c for c in module_build.up_to_current_batch()
         if not c.tagged and c.state == koji.BUILD_STATES['COMPLETE']
     ]
 

module_build_service/utils.py

@@ -152,12 +152,10 @@ def continue_batch_build(config, module, session, builder, components=None):
     further_work = []
     components_to_build = []
     for c in unbuilt_components:
-        previous_component_build = None
         # Check to see if we can reuse a previous component build
-        # instead of rebuilding it if the builder is Koji
-        if conf.system == 'koji':
-            previous_component_build = get_reusable_component(
-                session, module, c.package)
+        # instead of rebuilding it
+        previous_component_build = get_reusable_component(
+            session, module, c.package)
         # If a component build can't be reused, we need to check
         # the concurrent threshold.
         if (not previous_component_build
@@ -166,37 +164,7 @@ def continue_batch_build(config, module, session, builder, components=None):
             break
 
         if previous_component_build:
-            log.info(
-                'Reusing component "{0}" from a previous module '
-                'build with the nvr "{1}"'.format(
-                    c.package, previous_component_build.nvr))
-            c.reused_component_id = previous_component_build.id
-            c.task_id = previous_component_build.task_id
-            # Use BUILDING state here, because we want the state to change to
-            # COMPLETE by the fake KojiBuildChange message we are generating
-            # few lines below. If we would set it to the right state right
-            # here, we would miss the code path handling the KojiBuildChange
-            # which works only when switching from BUILDING to COMPLETE.
-            c.state = koji.BUILD_STATES['BUILDING']
-            c.state_reason = \
-                'Reused component from previous module build'
-            c.nvr = previous_component_build.nvr
-            nvr_dict = kobo.rpmlib.parse_nvr(c.nvr)
-            # Add this message to further_work so that the reused
-            # component will be tagged properly
-            further_work.append(
-                module_build_service.messaging.KojiBuildChange(
-                    msg_id='start_build_batch: fake msg',
-                    build_id=None,
-                    task_id=c.task_id,
-                    build_new_state=previous_component_build.state,
-                    build_name=c.package,
-                    build_version=nvr_dict['version'],
-                    build_release=nvr_dict['release'],
-                    module_build_id=c.module_id,
-                    state_reason=c.state_reason
-                )
-            )
+            further_work += reuse_component(c, previous_component_build)
             continue
 
         # We set state to BUILDING here, because we are going to build the
@@ -806,6 +774,96 @@ def module_build_state_from_msg(msg):
         % (state, type(state), list(models.BUILD_STATES.values())))
     return state
 
+def reuse_component(component, previous_component_build,
+                    change_state_now=False):
+    """
+    Reuses component build `previous_component_build` instead of building
+    component `component`
+
+    Returns the list of BaseMessage instances to be handled later by the
+    scheduler.
+    """
+
+    import koji
+
+    log.info(
+        'Reusing component "{0}" from a previous module '
+        'build with the nvr "{1}"'.format(
+            component.package, previous_component_build.nvr))
+    component.reused_component_id = previous_component_build.id
+    component.task_id = previous_component_build.task_id
+    if change_state_now:
+        component.state = previous_component_build.state
+    else:
+        # Use BUILDING state here, because we want the state to change to
+        # COMPLETE by the fake KojiBuildChange message we are generating
+        # few lines below. If we would set it to the right state right
+        # here, we would miss the code path handling the KojiBuildChange
+        # which works only when switching from BUILDING to COMPLETE.
+        component.state = koji.BUILD_STATES['BUILDING']
+    component.state_reason = \
+        'Reused component from previous module build'
+    component.nvr = previous_component_build.nvr
+    nvr_dict = kobo.rpmlib.parse_nvr(component.nvr)
+    # Add this message to further_work so that the reused
+    # component will be tagged properly
+    return [
+        module_build_service.messaging.KojiBuildChange(
+            msg_id='reuse_component: fake msg',
+            build_id=None,
+            task_id=component.task_id,
+            build_new_state=previous_component_build.state,
+            build_name=component.package,
+            build_version=nvr_dict['version'],
+            build_release=nvr_dict['release'],
+            module_build_id=component.module_id,
+            state_reason=component.state_reason
+        )
+    ]
+
+def attempt_to_reuse_all_components(builder, session, module):
+    """
+    Tries to reuse all the components in a build. The components are also
+    tagged to the tags using the `builder`.
+
+    Returns True if all components could be reused, otherwise False. When
+    False is returned, no component has been reused.
+    """
+
+    # [(component, component_to_reuse), ...]
+    component_pairs = []
+
+    # Find out if we can reuse all components and cache component and
+    # component to reuse pairs.
+    for c in module.component_builds:
+        if c.package == "module-build-macros":
+            continue
+        component_to_reuse = get_reusable_component(
+            session, module, c.package)
+        if not component_to_reuse:
+            return False
+
+        component_pairs.append((c, component_to_reuse))
+
+    # Stores components we will tag to buildroot and final tag.
+    components_to_tag = []
+
+    # Reuse all components.
+    for c, component_to_reuse in component_pairs:
+        # Set the module.batch to the last batch we have.
+        if c.batch > module.batch:
+            module.batch = c.batch
+
+        # Reuse the component
+        reuse_component(c, component_to_reuse, True)
+        components_to_tag.append(c.nvr)
+
+    # Tag them
+    builder.buildroot_add_artifacts(components_to_tag, install=False)
+    builder.tag_artifacts(components_to_tag)
+
+    return True
+
 def get_reusable_component(session, module, component_name):
     """
     Returns the component (RPM) build of a module that can be reused
@@ -818,6 +876,11 @@ def get_reusable_component(session, module, component_name):
     :return: the component (RPM) build SQLAlchemy object, if one is not found,
     None is returned
     """
+
+    # We support components reusing only for koji and test backend.
+    if conf.system not in ['koji', 'test']:
+        return None
+
     mmd = module.mmd()
     # Find the latest module that is in the done or ready state
     previous_module_build = session.query(models.ModuleBuild)\