fm-orchestrator/module_build_service/utils/reuse.py

# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
import kobo.rpmlib

import module_build_service.messaging
from module_build_service import log, models, conf
from module_build_service.db_session import db_session
from module_build_service.utils.mse import get_base_module_mmds
from module_build_service.resolver import GenericResolver


def reuse_component(component, previous_component_build, change_state_now=False):
    """
    Reuses component build `previous_component_build` instead of building
    component `component`

    Please remember to commit the changes where the function is called.
    This allows callers to reuse multiple component builds and commit them all
    at once.

    Returns the list of BaseMessage instances to be handled later by the
    scheduler.
    """

    import koji

    log.info(
        'Reusing component "{0}" from a previous module '
        'build with the nvr "{1}"'.format(component.package, previous_component_build.nvr)
    )
    component.reused_component_id = previous_component_build.id
    component.task_id = previous_component_build.task_id
    if change_state_now:
        component.state = previous_component_build.state
    else:
        # Use BUILDING state here, because we want the state to change to
        # COMPLETE by the fake KojiBuildChange message we are generating
        # few lines below. If we would set it to the right state right
        # here, we would miss the code path handling the KojiBuildChange
        # which works only when switching from BUILDING to COMPLETE.
        component.state = koji.BUILD_STATES["BUILDING"]
    component.state_reason = "Reused component from previous module build"
    component.nvr = previous_component_build.nvr
    nvr_dict = kobo.rpmlib.parse_nvr(component.nvr)
    # Add this message to further_work so that the reused
    # component will be tagged properly
    return [
        module_build_service.messaging.KojiBuildChange(
            msg_id="reuse_component: fake msg",
            build_id=None,
            task_id=component.task_id,
            build_new_state=previous_component_build.state,
            build_name=nvr_dict["name"],
            build_version=nvr_dict["version"],
            build_release=nvr_dict["release"],
            module_build_id=component.module_id,
            state_reason=component.state_reason,
        )
    ]


def get_reusable_module(module):
    """
    Returns previous module build of the module `module` in case it can be
    used as a source module to get the components to reuse from.

    In case there is no such module, returns None.

    :param module: the ModuleBuild object of module being built.
    :return: ModuleBuild object which can be used for component reuse.
    """

    if module.reused_module:
        return module.reused_module

    mmd = module.mmd()
    previous_module_build = None

    # The `base_mmds` will contain the list of base modules against which the possible modules
    # to reuse are built. There are three options how these base modules are found:
    #
    # 1) The `conf.allow_only_compatible_base_modules` is False. This means that MBS should
    #    not try to find any compatible base modules in its DB and simply use the buildrequired
    #    base module as it is.
    # 2) The `conf.allow_only_compatible_base_modules` is True and DBResolver is used. This means
    #    that MBS should try to find the compatible modules using its database.
    #    The `get_base_module_mmds` finds out the list of compatible modules and returns mmds of
    #    all of them.
    # 3) The `conf.allow_only_compatible_base_modules` is True and KojiResolver is used. This
    #    means that MBS should *not* try to find any compatible base modules in its DB, but
    #    instead just query Koji using KojiResolver later to find out the module to
    #    reuse. The list of compatible base modules is defined by Koji tag inheritance directly
    #    in Koji.
    #    The `get_base_module_mmds` in this case returns just the buildrequired base module.
    if conf.allow_only_compatible_base_modules:
        log.debug("Checking for compatible base modules")
        base_mmds = get_base_module_mmds(db_session, mmd)["ready"]
        # Sort the base_mmds based on the stream version, higher version first.
        base_mmds.sort(
            key=lambda mmd: models.ModuleBuild.get_stream_version(mmd.get_stream_name(), False),
            reverse=True)
    else:
        log.debug("Skipping the check for compatible base modules")
        base_mmds = []
        for br in module.buildrequires:
            if br.name in conf.base_module_names:
                base_mmds.append(br.mmd())

    for base_mmd in base_mmds:
        previous_module_build = (
            db_session.query(models.ModuleBuild)
                      .filter_by(name=mmd.get_module_name())
                      .filter_by(stream=mmd.get_stream_name())
                      .filter_by(state=models.BUILD_STATES["ready"])
                      .filter(models.ModuleBuild.scmurl.isnot(None))
                      .order_by(models.ModuleBuild.time_completed.desc()))

        koji_resolver_enabled = base_mmd.get_xmd().get("mbs", {}).get("koji_tag_with_modules")
        if koji_resolver_enabled:
            # Find ModuleBuilds tagged in the Koji tag using KojiResolver.
            resolver = GenericResolver.create(db_session, conf, backend="koji")
            possible_modules_to_reuse = resolver.get_buildrequired_modules(
                module.name, module.stream, base_mmd)

            # Limit the query to these modules.
            possible_module_ids = [m.id for m in possible_modules_to_reuse]
            previous_module_build = previous_module_build.filter(
                models.ModuleBuild.id.in_(possible_module_ids))

            # Limit the query to modules sharing the same `build_context_no_bms`. That means they
            # have the same buildrequirements.
            previous_module_build = previous_module_build.filter_by(
                build_context_no_bms=module.build_context_no_bms)
        else:
            # Recompute the build_context with compatible base module stream.
            mbs_xmd = mmd.get_xmd()["mbs"]
            if base_mmd.get_module_name() not in mbs_xmd["buildrequires"]:
                previous_module_build = None
                continue
            mbs_xmd["buildrequires"][base_mmd.get_module_name()]["stream"] \
                = base_mmd.get_stream_name()
            build_context = module.calculate_build_context(mbs_xmd["buildrequires"])

            # Limit the query to find only modules sharing the same build_context.
            previous_module_build = previous_module_build.filter_by(build_context=build_context)

        # If we are rebuilding with the "changed-and-after" option, then we can't reuse
        # components from modules that were built more liberally
        if module.rebuild_strategy == "changed-and-after":
            previous_module_build = previous_module_build.filter(
                models.ModuleBuild.rebuild_strategy.in_(["all", "changed-and-after"])
            )

        previous_module_build = previous_module_build.first()

        if previous_module_build:
            break

    # The component can't be reused if there isn't a previous build in the done
    # or ready state
    if not previous_module_build:
        log.info("Cannot re-use.  %r is the first module build." % module)
        return None

    module.reused_module_id = previous_module_build.id
    db_session.commit()

    return previous_module_build


def attempt_to_reuse_all_components(builder, module):
    """
    Tries to reuse all the components in a build. The components are also
    tagged to the tags using the `builder`.

    Returns True if all components could be reused, otherwise False. When
    False is returned, no component has been reused.
    """

    previous_module_build = get_reusable_module(module)
    if not previous_module_build:
        return False

    mmd = module.mmd()
    old_mmd = previous_module_build.mmd()

    # [(component, component_to_reuse), ...]
    component_pairs = []

    # Find out if we can reuse all components and cache component and
    # component to reuse pairs.
    for c in module.component_builds:
        if c.package == "module-build-macros":
            continue
        component_to_reuse = get_reusable_component(
            module,
            c.package,
            previous_module_build=previous_module_build,
            mmd=mmd,
            old_mmd=old_mmd,
        )
        if not component_to_reuse:
            return False

        component_pairs.append((c, component_to_reuse))

    # Stores components we will tag to buildroot and final tag.
    components_to_tag = []

    # Reuse all components.
    for c, component_to_reuse in component_pairs:
        # Set the module.batch to the last batch we have.
        if c.batch > module.batch:
            module.batch = c.batch

        # Reuse the component
        reuse_component(c, component_to_reuse, True)
        components_to_tag.append(c.nvr)

    # Tag them
    builder.buildroot_add_artifacts(components_to_tag, install=False)
    builder.tag_artifacts(components_to_tag, dest_tag=True)

    return True


def get_reusable_components(module, component_names, previous_module_build=None):
    """
    Returns the list of ComponentBuild instances belonging to previous module
    build which can be reused in the build of module `module`.

    The ComponentBuild instances in returned list are in the same order as
    their names in the component_names input list.

    In case some component cannot be reused, None is used instead of a
    ComponentBuild instance in the returned list.

    :param module: the ModuleBuild object of module being built.
    :param component_names: List of component names to be reused.
    :kwarg previous_module_build: the ModuleBuild instance of a module build
        which contains the components to reuse. If not passed, get_reusable_module
        is called to get the ModuleBuild instance.
    :return: List of ComponentBuild instances to reuse in the same
             order as `component_names`
    """
    # We support components reusing only for koji and test backend.
    if conf.system not in ["koji", "test"]:
        return [None] * len(component_names)

    if not previous_module_build:
        previous_module_build = get_reusable_module(module)
    if not previous_module_build:
        return [None] * len(component_names)

    mmd = module.mmd()
    old_mmd = previous_module_build.mmd()

    ret = []
    for component_name in component_names:
        ret.append(
            get_reusable_component(
                module, component_name, previous_module_build, mmd, old_mmd)
        )

    return ret


def get_reusable_component(
    module, component_name, previous_module_build=None, mmd=None, old_mmd=None
):
    """
    Returns the component (RPM) build of a module that can be reused
    instead of needing to rebuild it

    :param module: the ModuleBuild object of module being built with a formatted
        mmd
    :param component_name: the name of the component (RPM) that you'd like to
        reuse a previous build of
    :param previous_module_build: the ModuleBuild instances of a module build
        which contains the components to reuse. If not passed, get_reusable_module
        is called to get the ModuleBuild instance. Consider passing the ModuleBuild
        instance in case you plan to call get_reusable_component repeatedly for the
        same module to make this method faster.
    :param mmd: Modulemd.ModuleStream of `module`. If not passed, it is taken from
        module.mmd(). Consider passing this arg in case you plan to call
        get_reusable_component repeatedly for the same module to make this method faster.
    :param old_mmd: Modulemd.ModuleStream of `previous_module_build`. If not passed,
        it is taken from previous_module_build.mmd(). Consider passing this arg in
        case you plan to call get_reusable_component repeatedly for the same
        module to make this method faster.
    :return: the component (RPM) build SQLAlchemy object, if one is not found,
        None is returned
    """

    # We support component reusing only for koji and test backend.
    if conf.system not in ["koji", "test"]:
        return None

    # If the rebuild strategy is "all", that means that nothing can be reused
    if module.rebuild_strategy == "all":
        message = ("Cannot reuse the component {component_name} because the module "
                   "rebuild strategy is \"all\".").format(
                       component_name=component_name)
        module.log_message(db_session, message)
        return None

    if not previous_module_build:
        previous_module_build = get_reusable_module(module)
        if not previous_module_build:
            message = ("Cannot reuse because no previous build of "
                       "module {module_name} found!").format(
                module_name=module.name)
            module.log_message(db_session, message)
            return None

    if not mmd:
        mmd = module.mmd()
    if not old_mmd:
        old_mmd = previous_module_build.mmd()

    # If the chosen component for some reason was not found in the database,
    # or the ref is missing, something has gone wrong and the component cannot
    # be reused
    new_module_build_component = models.ComponentBuild.from_component_name(
        db_session, component_name, module.id)
    if (
        not new_module_build_component
        or not new_module_build_component.batch
        or not new_module_build_component.ref
    ):
        message = ("Cannot reuse the component {} because it can't be found in the "
                   "database").format(component_name)
        module.log_message(db_session, message)
        return None

    prev_module_build_component = models.ComponentBuild.from_component_name(
        db_session, component_name, previous_module_build.id
    )
    # If the component to reuse for some reason was not found in the database,
    # or the ref is missing, something has gone wrong and the component cannot
    # be reused
    if (
        not prev_module_build_component
        or not prev_module_build_component.batch
        or not prev_module_build_component.ref
    ):
        message = ("Cannot reuse the component {} because a previous build of "
                   "it can't be found in the database").format(component_name)
        new_module_build_component.log_message(db_session, message)
        return None

    # Make sure the ref for the component that is trying to be reused
    # hasn't changed since the last build
    if prev_module_build_component.ref != new_module_build_component.ref:
        message = ("Cannot reuse the component because the commit hash changed"
                   " since the last build")
        new_module_build_component.log_message(db_session, message)
        return None

    # At this point we've determined that both module builds contain the component
    # and the components share the same commit hash
    if module.rebuild_strategy == "changed-and-after":
        # Make sure the batch number for the component that is trying to be reused
        # hasn't changed since the last build
        if prev_module_build_component.batch != new_module_build_component.batch:
            message = ("Cannot reuse the component because it is being built in "
                       "a different batch than in the compatible module build")
            new_module_build_component.log_message(db_session, message)
            return None

        # If the mmd.buildopts.macros.rpms changed, we cannot reuse
        buildopts = mmd.get_buildopts()
        if buildopts:
            modulemd_macros = buildopts.get_rpm_macros()
        else:
            modulemd_macros = None

        old_buildopts = old_mmd.get_buildopts()
        if old_buildopts:
            old_modulemd_macros = old_buildopts.get_rpm_macros()
        else:
            old_modulemd_macros = None

        if modulemd_macros != old_modulemd_macros:
            message = ("Cannot reuse the component because the modulemd's macros are"
                       " different than those of the compatible module build")
            new_module_build_component.log_message(db_session, message)
            return None

        # At this point we've determined that both module builds contain the component
        # with the same commit hash and they are in the same batch. We've also determined
        # that both module builds depend(ed) on the same exact module builds. Now it's time
        # to determine if the components before it have changed.
        #
        # Convert the component_builds to a list and sort them by batch
        new_component_builds = list(module.component_builds)
        new_component_builds.sort(key=lambda x: x.batch)
        prev_component_builds = list(previous_module_build.component_builds)
        prev_component_builds.sort(key=lambda x: x.batch)

        new_module_build_components = []
        previous_module_build_components = []
        # Create separate lists for the new and previous module build. These lists
        # will have an entry for every build batch *before* the component's
        # batch except for 1, which is reserved for the module-build-macros RPM.
        # Each batch entry will contain a set of "(name, ref, arches)" with the name,
        # ref (commit), and arches of the component.
        for i in range(new_module_build_component.batch - 1):
            # This is the first batch which we want to skip since it will always
            # contain only the module-build-macros RPM and it gets built every time
            if i == 0:
                continue

            new_module_build_components.append({
                (value.package, value.ref,
                    tuple(sorted(mmd.get_rpm_component(value.package).get_arches())))
                for value in new_component_builds
                if value.batch == i + 1
            })

            previous_module_build_components.append({
                (value.package, value.ref,
                    tuple(sorted(old_mmd.get_rpm_component(value.package).get_arches())))
                for value in prev_component_builds
                if value.batch == i + 1
            })

        # If the previous batches don't have the same ordering, hashes, and arches, then the
        # component can't be reused
        if previous_module_build_components != new_module_build_components:
            message = ("Cannot reuse the component because a component in a previous"
                       " batch has been added, removed, or rebuilt")
            new_module_build_component.log_message(db_session, message)
            return None

    # check that arches have not changed
    pkg = mmd.get_rpm_component(component_name)
    if set(pkg.get_arches()) != set(old_mmd.get_rpm_component(component_name).get_arches()):
        message = ("Cannot reuse the component because its architectures"
                   " have changed since the compatible module build").format(component_name)
        new_module_build_component.log_message(db_session, message)
        return None

    reusable_component = db_session.query(models.ComponentBuild).filter_by(
        package=component_name, module_id=previous_module_build.id).one()
    log.debug("Found reusable component!")
    return reusable_component